{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型评估与改进"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#导入相关模块\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sc\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import mglearn\n",
    "%matplotlib inline\n",
    "\n",
    "#忽略弹出的warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')  \n",
    "pd.set_option('display.float_format', lambda x: '%.5f' % x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 交叉验证\n",
    "### 评估模型泛化性能常用的统计方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA10AAACnCAYAAADqmja9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzt3XucXHV9//HXEChgiY2Ika4pJUL3\nQ5GrUaLGarAgE7k0hq4BDWvEG+rKtGkxC/qzQqUKKnUkioJVhIja1TW1RiYRcUFug4CIiPmshlFZ\nEMNWlyiIQJjfH+fMelh2LsucM3Nm5/18PPYxuzPvOfudzCdn9nMu35Mpl8uIiIiIiIhIMnZq9wBE\nRERERERmMzVdIiIiIiIiCVLTJSIiIiIikiA1XSIiIiIiIglS0yUiIiIiIpIgNV0iIiIiIiIJUtMl\nIiIiIiKSIDVdIiLSEDNbbWbfDL//rJkdNU3mRWb28waW9RYze2f4/WlmNhj3eEVERNJi53YPQERE\nOo+7v6XJRbwcuDNc1qebH5GIiEh6qekSEWkxMzsV+BdgBzAOfB74V+AhYA/gxcAbgdPDzK+BAXcf\nNbOXAxcAc4Ay8CF3/1q1+2uM4W3A8e5+fPjzAcB3gH3C3/124M+APYEPu/tFU54/Aqxz96+a2TuA\nfwYeBH4UyTwX+AzwXGBv4BfA64AlwAnA0Wb2B+A5wF7uPmBmLwDWAc8OX8fH3P0yM1sKnAvcDRwE\n7AK83d2vb+TfXEREpJ10eKGISAuZ2aHAeUDW3Q8BvgG8l6CRODm8bwnwHuBIdz8UuALYYGYZ4Gzg\nAndfBJwKvCpcdLX7q/kS8HIz2zv8+U0Ezd/uwFuB17j74cBK4Pwar+cw4APAK9z9xcCjkYdPAm50\n95cCzwceBk5x96+Hr/s/3f2TkWXtHN5/YfjvsAz4DzN7aRhZTNCEHR6O9T/qvEYREZFUUNMlItJa\nfw9scvd7ANz948BpwD3u/oswkwW+4u4PhJlLgecB+wL/DXzSzL4ILALOCp9T7f5pufvvgGFglZnN\nAd4A/Je7/x44DjjWzP6doCHco87r2ezu94c/Xxz5HXngBjNbA3yKoLGstaxeYDd3Hw6ffx/wtfDf\nA+AX7n57+P1tBHvhREREUk9Nl4hIaz1OcNgcAGa2O3AA8PtIZk40E8oAu7j7Z4CDgW8DxwB3mNlu\n1e6vM5ZLgH6CpuYn7l4yswXA7cBfA9cB72vgNWWmvL7KazsPOAd4gKAZ2zwlO9V0r3sngkMJAf4Q\nub9cZ1kiIiKpoaZLRKS1vgscZWZ/Gf78dp56+F4BOMnMngNgZm8C/g/4mZndABwe7v16GzAP2Lva\n/bUG4u43ETQu7ydowABeRNAkfZCgSTouHMOcKovZDLw6bNYAVkceOwb4uLtfDmwDjiZorCBoznbh\nybYAj5nZivB39gAnEjSSIiIiHUtNl4hIC7n7j4AzgIKZ/ZBgL9NpUzLfBv4TuNrMfkwwscVx7v4E\nwble55jZD4AR4Gx3/3mN++u5hOB8qw3hz5uBMcCBnxBMrPEAsH+N1/Me4DtmdgsQ3bt2DvBRM7uD\n4Fyt6yLLuRI4zczOjCzrMWA5kAufcxVwjrt/t4HXISIiklqZcnnqkRwiIiIiIiISF00ZLyIyS5nZ\nGQQTZEznI+7+xVaOR0REpFtpT5eIiIiIiEiCdE6XiIiIiIhIgtR0iYiIiIiIJEhN1+x3Z7sHIG2n\nGhDVgKgGRDUg0kY6p2v20wVERTUgqgFRDYhqQKSNtKdLREREREQkQWq6REREREREEjTrr9OVyWR6\n2z2GdnJ3zKwb/w22lsvlHZlMZg6wX7sH006qAdVAN9dAeNvV7z+oBlANdGsNbC2XyzvaPQgR6IJz\nuo4++ujyddddRzabZcGCBXXzY2NjFAoF5Ts4PzExwfr1661cLo9mMpneQqHgCxcuBKBYLJLL5cjn\n8yxevLju8pXvzPzg4CBr166dtgY6YfzKN58vlUpks1kDOO+883z58uUdNX7lm89Ha6BQKPi2bds6\navzKN5evvP/lcnm07pNFWmDWN1277757eeXKlUz3B9dUpVKJoaEh+vr6lO/g/Pj4OOvWrZv8g9vd\nvbe3l5GREfr6+hgaGmLp0qV1l6985+Z7enows6fUQKeMX/nm85dffjn9/f0GMN37n/bxK998fnR0\nFDMzgMsuu8zXrFnTUeNXvrl85f1X0yVpMevP6cpms6luEJRvTb4TPiCUV175+PK5XK5urpXjUb59\neYBcLpea8Sjf3rxIuyTWdJnZoJldZWabzWyTmS2qkV1qZl8Ovx8Obw82s1dUyT/HzH5qZrvVG0cj\nh6ilrUFQPt58sVhM1QpfeeWVTz6fz+frZls5HuXbk6/I5/OpGI/y7c2LtFMiTZeZHQicABzt7q8G\n1gKfa+S57r4i/PZE4MBpln0MsBl4bhxjTVuDoHy8edAWTuWV78Z8I+eApHn8yjefLxaLk9+rHpQX\nabekZi/cBuwDnGpmBXe/3cyOADCzEWALcADBRfpWRp9oZvcDi4DVwKNmdpu73xyJPAEcBdza7CDT\n1iAoH2++Qls4lVe++/Kjo7VP40j7+JVvPq9DTJUXSZNE9nS5+zjBnq4lwI1mtgU4LhK5wd2XAl8B\nzprm+fcClwIXTGm4cPdvu/v/NTvGtDUIysebd3cHyu7u2sLZnfloDegQ0+7Meygt41G+tfl8Pl+1\nBjph/Mo3n698DrT4S2RaiezpMrP9ge3ufmr484uAb5nZd8PI1eHtDcA/JDGGWtLWICgfb35sbGxy\nxiIz69UHbnfmozXwrGc9y4eHhztq/Mo3n6/MXBddB3TS+JVvLl+ZxRSeXAOdMn7lm89r9kJJk0T2\ndAGHABdFJroYBR4EKheoq0yqsQT4cZVlPJHE+NLWICgff75QKNTNVaTtA0L5ePMVOsRUeeWVV767\n8tFz+kTSIKnDC4eBEaBoZtcDm4Az3P3BMLLazK4BjgXOrbKYW4EBMzsyrnGlsUFQPv58Nputm4X0\nfUAoH28+SoeYKq+88sp3V34m5/SJtEJSE2ng7udSvaE60923RH4eCb9w973D243AxhrL33cm40lr\ng6B8/Pm5c+fWzafxA0L5+PIz3cKZtvErH2++WCyiC+Mqr3x35fP5PP39/XXzIq2SWNOVJmluEJSP\nPz8+Pl4zn9YPCOXjy2vWMuWjcrkcOqdPeeW7K9/T01M3L9JKLW+6wlkLW2bLli1cd911ZLNZ5s6d\nW/cP8rGxMQqFgvIdnJ+YmHhSplQqTX5fLBbJ5XLk83l6enrqTiutfGfmBwcHWbt27eT90RrohPEr\n33w++p4PDg62fTzKtz4frYFSqdT28Sjf2ny19b5Iu2TK5dk9u2Umk+lt9xjayd29MntTl9laLpd3\nZDKZOcB+7R5MO6kGVAPdXAPhbVe//6AaQDXQrTWwtVwu76gfE0nerG+6hDLBRaile6kGRDUgqgFR\nDYi0USKzF4qIiIiIiEhATZeIiIiIiEiC1HSJiIiIiIgkSE2XiIiIiIhIgupOGW9m+wEvAa4APgMc\nDrzD3W9JeGwiIiIiIiIdr5HrdH0euAQ4AegF1gCfAF6W4LhioynjHTPrxn8DTRceUg2oBrq5BsLb\nrn7/QTWAaqBba0BTxktq1J0y3sxudvcjzOyzQNHdLzGzW919UWuG2JxNmzaVFy5cCDz5wnmLFy+u\n+1zlOzMfXhjXyuXyaCaT6V21apXPmzfvKfnohZUXLFhQd/nKd05+YmKC9evXT9ZAoVDwynqgmrTW\ns/JPL18qlchmswZQKBR827ZtHTV+5ZvPT62B6DqgE8avfHP5gYEBtm/fbuVyufZVlUVapJGm60bg\no8AnCQ4tPAJ4r7sfkfzwmjc6Olru7e1lZGSEvr4+hoaGWLp0ad3nKd+5+Z6eHsxs8g/ugYEB32uv\nvZ6UL5VKDA0N0dfXR70/xpXvvPztt9/Ohg0bJmvA3b23t/oG3jTXs/JPLz86OkrlQrCXXXaZr1mz\npqPGr3zz+WgNRNcBnTJ+5ZvLX3DBBfT396vpktRoZCKNtwHHAu9y918BJwNvSXRUMUvrCkH59uTT\n1iAoH3++UCjUzVWkrT6VjzcPkMvlUjMe5ZVXvjX5RvaIibRS3abL3X8E/BNws5ntA7wHmEh6YHEp\nFoupXSEo3/p8GhsE5ePPZ7PZullIX30qH2++Ip/Pp2I8yiuvfHryIq3WyOyFZwGDwP9F7i4Dz6/z\nvEHgKOCJMH+Wu99aJbsUOM3dTzKzYXdfYWYHA89y92unZP8ZOCn88VvufnatceRyOYaHh1PxH1z5\n9ubT2iAoH39+7ty5dfNpq0/l480Xi8XJ7xvZ4p228Ssfb75YLKJDTJUXaadGZi98M7Cfuz/Q6ELN\n7ECC2Q6XuHvZzA4DvgAcWu+57r4i/PZE4H5gsukys+cDbwAWEzRy3zOzr7v7HdWWpy2cykO6GwTl\n48+Pj4/XzKetPpWPP5/L5ermWjke5duXB22AVV6k/Rppun4J/GaGy90G7AOcamYFd7/dzI4AMLMR\nYAtwAJABVkafaGb3A4uA1cCjZnabu98cPnwPkHX3HWF2F+CRWgPRFk7lx8bG2Lx5c2obBOVbm09b\nfSqfTD6fz9Pf35+a8SjfnnyFNsAqL9JujTRdPwWuM7PvEmlw3P2cak9w93EzOwEYAP7NzB4G3gt8\nLYzc4O6nmdk7gbOA4SnPv9fMLgXujzRcuPtjwLiZZYCPAD9w96ZmpUnbCkH5ePLu7uEthx56KCtX\nrkzFH/zKty4frYGKtNSn8snne3p6JmsgDeNRvvX5YrFYtwZaOR7lW5+fyfsfo0wbfqd0gJ0ayNwL\nFIA/EhRS5asqM9sf2O7up7r7PsAq4CIz2zOMXB3e3gDYTAZsZrsBXwTmAu+cyXOnSsMKQflk8uE0\nwRkzs2w2m/oGQfn489EagHTVp/KtyVsoLeNRvrX5XC5XswbSPn7lm89XPgda/CUyrbpNVzhRxaeA\nW4EfAp+uN3kFcAhBk7Vb+PMo8CBQuSp45cLKS4AfV1nGE1PHF+7h+h/gh+7+9sphhk9HWlYIyid/\nCEEjF95td4OgfLJ5zWKqvPLdl8/n83WzrRyP8u3Ji6RF3abLzI4BbgfeBLwRuMPMjqv1HHcfBkaA\nopldD2wCznD3B8PIajO7huD6X+dWWcytwICZHRm5bznwSmCZmY2EXy+t9xqmStsKQfl489FZyxqR\ntgZB+XjzoOs0Ka98N+Z1TrfyImnSyDld5wIvd/cSTM4gOAx8s9aT3P1cqjdUZ7r7lsjPI+EX7r53\neLsR2DhlmV8HdqMJaVshKB9/fiazlqWtQVA+3nyFTqJXXvnuy4+O1j7lO+3jV765/Ew3wM5WmUxm\nDrBfzIvdWi6Xn/bRZt2qkaZrl0rDBeDud5tZ3T1kaVEqTQ6dYrFILpcjn8/T09NTd4WsfGfmBwcH\nWbt27eT9ExPTX8t7bGyMQqFANptl7ty5dacZV75z8lPf8/nz56emPpVvTT667i+VSm0fj/Ktz0+t\ngXaPR/nW5gcGBmpmush+hULBoxsqo/+ejewRjubnz59PNps1glOHpmVmHyM4lWhv4BnA3cAD7t5X\n73eFl5k6odqEfWaWBfZx94vrDrwBZnYTcJK7/7zK468AJmpdnqpRmXK5XG8w/wt8B/iv8K63AK9y\n9+Ob/eWtkMlkets9hnZyd2/kJOJZaGu5XN6R0BaejqIaUA10cw2Et139/oNqANVAt9ZA1++RyWQy\nve7uvb3Bn8Nx7EE2MyuXy3VnDzez1cAB7j7Y5MtITANN16XAl9290OzvavTiyBcSTPm+E0ED9rZm\nf3GrNFIUs103/xuEK9uuff0VqgHVQDfXAHr/AdVAuweQBl1eA12v3efQmdlS4DzgUeBi4A/Au/jT\nrI//CBwEnObuJ5nZT4HrCWY6/zVwInAKwbV+Pw18ieAavvsBN7v7O8xsL+AKYFfACXYU7T9lHOcC\n2fC5e4X3LQAuIjiN6dnAOeHjWeCFZnYXcAKwAtiFYILAFe7+aKOvv27T5e7bmHIBYxERERER6Qzt\nbrgidnP3xQBmdhZwrLs/bGafAY4huFRVxfMJmqZ7won5XjxlWb3Aq4GHgbvNbG9gLbDB3T9lZkeH\nj08ys4OAV4TL2oPgesQQNHIfc/cRM3sZcLa7H21mBeDLwBhBM3aUuz9hZpvCZVzf6Auv2nSZ2Tfd\n/TgzKwFPOQbR3Z/f6C8REREREZHWKxaLrFmzJg0NFwR7nyq2AV8ws98TND03TsmOu/s94ff38NTJ\n9H7m7r8DMLNfhY//LfCF8PHvTfP7XwDc4u5PANvN7Efh/b8C3mdmbyboe3Z50qCDRutR4EvheBdM\nzdRTa0/XW8PbpTNZoIiIiIiIpEMul2N4eDgNDRcE1+HFzP4COBvYJ7z/2zz14tK1J56Y/vE7gZcS\nXO7qJdM87sDp4aSAuwMHhvf/O3CJu19pZm8CVkfGu5OZHQIsd/fFZvYMgktbzehi2FWbLnf/Vfjt\nBe5+YvQxM/sO8Pcz+UUiIiIiItJag4ODsc0SOXUm0CZsJzg07zbgIeC3QA/Q7C/4MHC5mb0OuA94\nLPqgu99uZkPA98PHt4UPDQGfMLP7iZzrBRTDZZ4MPGRmtwB/JNgz1jOTgVWdvdDMhoHDwgXeF3lo\nZ+Aed18yk18kbVNmhp24zDqqAVENiGpAVANdqNuu02VmryGYnv77ZnYUcJa7v6rd44LahxeuBvYE\n8sDpkfsfJ5hBREREREREUqoLZ/AtAZ8zs8eBOTy5h2mrutfpAjCzwwlm+MgQvICF7v65hMcWC12n\nqyuvywG6RtMk1YBqoJtrILzt6vcfVAOoBrq1BlK7R0a6TyMXR76EYDKNPYGfEBxyeL27H5P46GKw\nadOmcvQq3NDclbiVT39+w4YNrF271srl8mgmk+mdeiX2tI9f+ebzpVKJbDY7WQO77rqrL1u2jAUL\nFtRd/tjYGIVCgWw2q3wH5ycmJli/fr0BRNcBaahP5VuTnz9/Ptls9ik10CnjV765fPRzoO6CRVqg\nkaarRDAP/oXAJ4BnEEyu8Yrkh9e80dHRcuUq3JD8dQqUb39+xYoV/Pa3v538gzt6JfZOGL/yzedH\nR0cxs8kaWL58uR922GF1l18qlRgaGqKvr49ajbry6c+Pj4+zbt06g2ALf29vb2rqU/nW5CvrAfhT\nDXTS+JVvLn/55ZfT39+vpktSY6cGMve5+2MEe7kOcfdbgL9IdljJSNsKQflk8vl8vm62leNRvj35\nqEb2mHRSQ6H8zPNpq0/llVc+2Xwul6ubE2mlWhNpVNxrZmcCVwHnhxuNdq33JDMbBI4imN++TDB7\nyK1VskuB09z9JDMbdvcVZnYw8Cx3v3ZK9l0Ek3yUgXPc/ZsNvIZUrhCUTybf01N/Bs80j1/55vPF\nYrFuJiptDYLy8eaTvjCo8sorn758Pp+nv7+/bn6267bZC9OskabrzcCx4dSLwwTz1L+j1hPM7EDg\nBGCJu5fN7DCCq0MfWu+XufuK8NsTgfuByabLzPYC3klwXtluwF1mttHdax4jmdYVgvLJ5OtdhyLt\n41e++fxMtnCmrUFQPt48JHthUOWVVz6d+UY2wHaJ/VatWuXz5s2L5ZzZyPmyVf/YMrOPAYuAvQlO\nS7qbYBr3vkYHbWb7AgfV2rliZl8FPu7u11V5/FBgbrXHW62RpuurlUkz3P1CgnO76tlGcIXpU82s\nEF6I7AgAMxsBtgAHEMyGuDL6xPCiZIsI9mY9ama3ufvN4e8fN7ND3f3x8M2YqNdwaQun8sp3X77R\nLZxpaxCUjzdfkc/nU1WfyiuvfPL5ehtgu8m8efP43e9+x+bNm1m5cmXD69uZ5KPc/V8AzGw1cIC7\nDz6NYR8F7As0dERbFX3Az4GOabqeYWZ/5e73NLrQsDk6ARgA/s3MHgbeC3wtjNzg7qeZ2TuBs4Dh\nKc+/18wuBe6vNFyRxx43swHgbIKJPWrSFs7uzLu7h7epGI/yrd/CGa2BCy986raitDUIysebHxsb\nm6yBRqS5npV/+vlKDWgDbHfmZ7IOiFHqLkA9NjbG5s2bE9/A1QgzOx94GcElqD7i7sNmdjrwBoJT\nkq4B3g+cAexmZje6+8bI808H3gT8imBPGmY2D7iEYM6JvYCLgE3AKcAjZvYDYH/gNIL3pwyc6O6/\nifXF1bFTA5m9gJ+b2a/M7G4zK5nZ3bWeYGb7A9vd/VR33wdYBVxkZnuGkavD2xuAGV8zwt3XAX8J\nvMLMjqyV1RbO7syHM1ZlKjNXtXs8yrc+P7UGotLWICgff75QKGChevlOqGfln16+UgO5XC4V41G+\ntfnK50CLv1KnUCikpeE6Hnieu78ceBXwATObS9BEvYugGfslsAP4CHD5lIbr2QSnGS0GlvOnOSb2\nB77o7q8O71/j7r8ELido7G4NM8vcfSnB4Y5Hx/riGtDInq7s01juIcA7zOx4d3+E4LjPBwn+ESE4\nfHAMWAL8uMoynmBKUxj+5/kQwflejwF/DHNVNXLdh05agSg/87y2cCoflcYGQfn489lslg0bNtTN\np60+lY83X6ENsMp3s2w22/aGK3Qw8OLwVCMI+pC/BvqBfyU4nPB6qjevBwA/cvdHAczs++H9vwZO\nN7MTgd8Du0zz3AeAy83s98CBwHebfTEzVXdPl7v/gqA5ehvBgF8Z3lfrOcPACFA0s+sJdvGd4e4P\nhpHVZnYNcCxwbpXF3AoMRPdkhbuJfwjcSLCX7CZ3v6bea6glbSsE5ePNQ3CIaVrGo3x782ltEJSP\nP9/ISeJpq0/l481HZzHVBljlu1kSl015mrYAV4V7m/4eGAJKwFuBt7n7Kwn2Yi1mmp0vwFbgIDPb\n1cx2Bg4P7z8D+J67n0JwylKlaXsC2Ck80u59BPNIvBV4hDbslay7p8vMPgwsINg7dR7wpnAyi3+p\n9Tx3P5fqDdWZ7r4l8vNI+IW77x3ebgQ2Tn2iu59NcD5X09K2QlA+3nyFtnAqD+luEJSPPz8+Pl4z\nn7b6VD7+/ExmMU3j+JWPL9/tJiYmaj4enaVw7ty5Ndef9ZZVx9eBV5rZ94A9CCbre8jM7gJuMbMH\ngHuAWwiOZltrZj9w9yEAd7/fzP4duIlg0r7t4XK/AXzSzN5IsIOobGZ/Fi7nwwTN3veB24CHgQmg\n5dNbNnJ44THAC4Hb3H27mR0N3AHUbLrSLm0rBOXjzUdpC6fySZ9ErHxn5dNWn8onk290FtO0jl/5\nePLC1nCK97oaOSS7ssxGQu5+6ZSfy8BTtoa4+6eBT0+5+xammffB3a8Arpjm1033Gr8RfkHkElTt\n0kjTVTlnqjI1+67UOY+qlnCXYsuUSqWn3FcsFsnlcuTzeXp6eupOK6p8Z+WnrjSmq4E0j1/55vNT\n3/Mrr7ySZcuW1d2CBzPb4qd8evPRrbHRekhDfSrfmvz8+fMn76/2OZDm8SvfXL7eZ3+3CC9irPnz\nUyBTLte8zBVmtpbg0MIjgI8TTL84HB4+mHqZTKa33WNoJ3f3RmbvmoW2lsvlHQldib2jqAZUA91c\nA+FtV7//oBpANdCtNbA1bDpE2q5u0wVgZscQXKRsDnB1ratDS+qUSekUptIyqgFRDYhqQFQDIm3U\nyJ6uC9393VPu+4K7vzHRkUlctJIV1YCoBkQ1IKoBkTaqek6XmX0WeD7wIjN7wZTnzEt6YCIiIiIi\nIrNBrYk0PkhwkbI8T56i/XHgJwmOSUREREREZNaodXHkR9x9BDie4MJlla97CObWFxERERERkTpq\n7en6LHAccA1PPQ64THDooYiIiIiIiNTQ0OyFnUxTxnflFLGg6cInqQZUA91cA+FtV7//oBpANdCt\nNaAp4yU1Zn3TtWnTpvLChQurPh690N7ixYvrLk/59OdLpRLZbNbK5fJoJpPpfeYzn+nr1q3rmPEr\n33x+ag0UCgVfuHBhx4xf+ebz8+fPJ5vNGsCqVat83rynzv8UvbDyggUL6i5f+c7Kb9myhauuusoA\nKuuAWtJcz8rPPB/9HKi7MJEWmPVN1+joaLm3d/qdXSMjI/T19TE0NMTSpUvrLkv5zsiPjo5iZpN/\ncF922WV+yimndMz4lW8+P7UG3N3vu+++jhm/8s3nKzUAMDAw4HvttdeT8qVSiaGhIfr6+qj3x7jy\nnZn/yle+wiOPPGIQ7OWp9rcApL+elZ95Pvo5UHeBIi1QdSINM9uzlQNptTSsEJRPLh/VyBaztI1f\n+XjzxWIxVeNRvr35NDYIysefz2azdbOQvvpUPt68SFrUmkhj1My+A3wO2Ozus2aXWNpWCMrHmy8W\ni3UzrRyP8u3NA+RyOYaHh1MxHuXbm09rg6B8/Pm5c+fWzaetPpWPNy+SJrWmjN8H+F9gDXC3mZ1j\nZvXXeCEzGzSzq8xss5ltMrNFNbJLzezL4ffD4e3BZvaKKvmdzOxKMzut0fFUpG2FoHz8+VwuVzfX\nyvEo3758RT6fT8V4lG9vPs0NgvKtz6etPpWPNz/TDbAiSavadLn7w+6+3t2PAV4GbAe+bmbfMbPX\n11qomR0InAAc7e6vBtYS7DGry91XhN+eCBxYJfZBYMaHP6ZthaB8Mvl8Pl8328rxKN+efJQOMVU+\nbX/wK9/efNrqU/n48zPZACvSCrUOL5zk7r8CPhrujXov8HngihpP2Uawp+xUMyu4++1mdgSAmY0A\nW4ADCK79tTL6RDO7H1gErAYeNbPb3P3myOP/CDwBXNnI2CvSuEJQPpl8T09PqsajfOvzOsRU+aix\nsTE2b96cmj/4lW9vPm31qXwy+Xw+T39/f928SKvUOrwQADObZ2ZvDZulbwO/BGqu1dx9nGBP1xLg\nRjPbQnCh5Yob3H0p8BXgrGmefy9wKXDBlIbrIOD1wPvrjTsqrSsE5ZPLu7sD5fC27eNRvrX5XC5X\nswbSPn7l48l7qFAopOYPfuVbm6/UQOXnNNWn8snmFy9ePPk50OIvkWlVbbrM7HVm9g3gZ8BLgP/n\n7n/r7h9y9/tqLdTM9ge2u/up7r4PsAq4KDIj4tXh7Q3ATC7U1w88L3z+amCNmdWcnkizlnVnPpwq\nOjP1QpCdMn7lm8vn8/mqNdDD8G0lAAAOnElEQVQJ41c+nryFstlsRzQIysefr9QApK8+lU8+X/kc\naPGXyLRqHV74boLzsE5294dmuNxDgHeY2fHu/ggwCjwIVK4KvggYI9gT9uMqy3iCKU2hu7+n8r2Z\nfQC4390LtQaiWcuUV7778jrEVPnoIaaNXEg3DQ2C8snli8Uia9asSU19Kt/evEg7VN3T5e5/B2wA\ndq/cZ2avNLPn1Fuouw8DI0DRzK4HNgFnuPuDYWS1mV0DHAucW2UxtwIDZnZkIy+kGs1aprzyyivf\nffmZnESftgZB+XjzEGyATVN9Kt++vEi7VN3TZWaHA98C3gRU9ia9GrjCzJa5+x21Fuzu51K9oTrT\n3bdEfh4Jv3D3vcPbjcDGGsv/QK3fX6FZy5TXFk7lle++fKMn0aetQVA+3nyFNsAqL9JutQ4v/CjB\noYUjlTvc/b1mdi1wAXBUwmOLRalUqvl4sVgkl8uRz+fp6elhdHRU+Q7PT33PBwYGWLduXceMX/nm\n81NroPJzp4xf+ebz8+fPn7x/YmJi2vzY2BiFQoFsNsvcuXMZHx+vuXzlOyu/Zcuftu3Onz8/VfWp\nfPL5en//ibRaplyefqKVcKr2F1Z57HZ3PyzRkcUkk8n0tnsM7eTuXmsigVlsa7lc3pHJZOYA+7V7\nMO2kGlANdHMNhLdd/f6DagDVQLfWwNZyubyjfkwkebWarh8Bh7r7E1Pu3wm4092rXbhY0qWMZtPp\ndqoBUQ2IakBUAyJtVHUiDeAa4N+muf99wC3JDEdERERERGR2qbWnay7BRBp/BdwOPAK8ENgGnODu\nv2nVIKUp2rIlqgFRDYhqQFQDIm1UtekCMLMMcCRwOMF1s25x9++1aGwSD61kRTUgqgFRDYhqQKSN\najZdMitoJSuqAVENiGpAVAMibVTrnC4RERERERFpkpouERERERGRBNW6OPKsoOt0OWbWjf8GukZT\nSDWgGujmGghvu/r9B9UAqoFurQFdp0tSY9af03X++eeXly9fXjcXvbL54sWLle/gfKlUIpvNWrlc\nHs1kMr2FQsG3bdvWMeNXvvn8dDWwcOHCjhm/8s3nBwYG2L59uwFMff87YfzKN5+vrAcAdt11V1+2\nbBkLFiyou/yxsTEKhQLZbFb5Ds5PTEywfv16K5fLo3UXItICs77pGh0dLff21t6wMzIyQl9fH0ND\nQyxdurTuMpVPd350dBQzm/yD+7LLLvM1a9Z0zPiVbz4/tQbc3SvrgU4Yv/LN5y+44AL6+/sNIPr+\nd8r4lW8+X1kPACxfvtwPO+ywussvlUoMDQ3R19dHrUZd+fTnx8fHWbdunZouSY1Zf3hhPWn6gFA+\n/jxALpdjeHg4FeNRXnnlW5Pv6elJ1XiUb32+WCxOft/IHpNOaiiUn3lepN0Sa7rMbBA4iuD6XmXg\nLHe/tUp2KXCau59kZsPuvsLMDgae5e7XTsl+AlgC/C686x/c/cGnM8a0fUAoH2++Ip/Pp2I8yiuv\nfOvyo6O1N26nffzKN5/P5XJ1cxVpaxCUjzcvkgaJzF5oZgcCJwBHu/urgbXA5xp5rruvCL89EThw\nmsgLgWPcfWn4pYZL+ZoaOUcgbeNXPt58sVhM1XiUV1755PP5fL5uFtLXICgfb14kLZLa07UN2Ac4\n1cwK7n67mR0BYGYjwBbgAIKL9K2MPtHM7gcWAauBR83sNne/OXxsJ+BvgIvN7LnAf7l7Q81cVFo/\nIJSPJx89pKQRaRu/8vHmQYeYKq98N+YbOcQ0bQ2C8vHmRdIkkT1d7j5OsKdrCXCjmW0BjotEbnD3\npcBXgLOmef69wKXABZWGK/TnwIXAKiALvNPMDpnJ2NL8AaF8PPlcLoe7O1AObztq/MrHk4/WgA4x\n7c68h9IyHuVbn59aA1FpaxCUjzc/NjY2+TnQ4i+RaSV1eOH+wHZ3P9Xd9yFoki4ysz3DyNXh7Q2A\nzWDRDwN5d3/Y3X8XLufQRp/cCR8Qyjefz+fzlRmrMpWZqzpp/MrHk4/WgA4x7c68hdIyHuVbn4/W\nQFTaGgTl488XCoXJz4EWf4lMK5GmCziEoMnaLfx5FHgQqFygblF4uwT4cZVlPDHN+HqB68xsjpnt\nArwcuK2RAXXKB4Tyzef1B7byOsRU+Sid06d8VBobBOXjz2ez2bpZkVZK6vDCYWAEKJrZ9cAm4IzI\npBerzewa4Fjg3CqLuRUYMLMjI8v9CfBF4CbgGuAyd6/WtE1K2wpfeeWVTzY/k1nL0jh+5ePLQ3BO\nX1rGo3x782ltEJSPP9/IZQJEWimxKePd/VyqN1RnuvuWyM8j4Rfuvnd4uxHYOM1yzwfOb3QcxWIR\nXRhXeeW7K5/P5+nv70/NeJRvT75C5/QpD+luEJSPPz8+Pl73OSKtNOsvjqxZy5RXvvvyujCu8tFD\nTHXIsfJjY2Ns3rw5tQ2C8q3Ni7RDy5uucNbClhkcHKSnp6fuhTKLxSK5XI58Pq98h+dLpdKTsqVS\nqaPGr3zz+elqoJPGr3zz+YGBgcmfp77/nTB+5ZvPR9/3K6+8kmXLljF37ty6e0DGxsYoFApks1nl\nOzg/MTFR83kirZYpl2f37JaZTKa33WNoJ3f3WjP4zWJby+XyjkwmMwfYr92DaSfVgGqgm2sgvO3q\n9x9UA6gGurUGtpbL5R31YyLJm/VNl1BGU5h2O9WAqAZENSCqAZE2SmT2QhEREREREQmo6RIRERER\nEUmQmi4REREREZEE6ZwuERERERGRBGlPl4iIiIiISILUdImIiIiIiCRITZeIiIiIiEiC1HSJiIiI\niIgkSE2XiIiIiIhIgtR0iYiIiIiIJGjndg9AGmNm+wJ3ALdF7r7a3c+pkr8U+LK7FyL37QZscfd9\np8k/A/g28GZ33xLfyCUuSdaAmZ0M/BOwI/wd73T3J+IcvzQv4Ro4ERgEysDF7v7ZWAcvsUj6syB8\n/GLgN+4+GNOwJUYJrwfWAG8GHgjveru7e2yDF+liaro6y13uvjTuhZrZi4BPAwviXrbELvYaMLPd\ngQ8CB7v7w2b2JeA44Btx/h6JTRI1MAf4MPAi4PfAXWa2wd3H4/w9EptEPgsAzOztwMHANUksX2KT\nVA28EOh391sTWLZIV1PTNQuY2ceAl4c/XuHu+chjewBfBJ4F/KzKInYFXgtcnuQ4JTlN1sAfgZe5\n+8PhzzsDjyQ4XElAMzXg7jvM7G/d/XEzmw9kCJov6SDNfhaY2UuBlwCfAQ5IdrSShBj+HlgEnGlm\newMb3f1DSY5XpJuo6eosB5rZSOTnNwCHAwsJPih3Bq4zs6sjmdXAne7+XjNbDLxq6kLd/XoAM0to\n2BKj2GsgPIzw1wBm9m5gD4JDTSWdkloPPG5mK4BPAhuBx5IZvsQg9hows78EPkCwAe51iY1c4pLI\negD4MsE6YDvwdTM7zt2/mcD4RbqOmq7O8pTDCczs9cD33L0MPGZmNwEHRiIvAAoA7l40M/0h1dkS\nqQEz2wk4H+gFTgyXJemU2HrA3YfNbANwKdAPfD7+4UsMkqiBPmAv4FvA3sAzzGyLu1+azEuQJsVe\nA2aWAT7u7g+GP28kaOTUdInEQLMXdr6fEB5KYGa7AC8Dfhp5fAvw0vDxw4FdWj1ASVwcNfAZYDdg\neeQwQ+kcTdWAmT3TzK4xs13DPZ8PAZpIpbM0VQPu/gl3XxT+If9hgkPTLk1+2BKjZj8LngncaWZ7\nhA3YqwCd2yUSEzVdHS7c7V8ysxuBm4Cvunt0RqNPAs8zs+uAdxGcvyOzSLM1YGYvJJit6mDgajMb\nMbPXtmb0Eodma8DdtxOc63FtmCkD61syeImFPgskhvXAg8BZwHeB7wE/dvdvtWTwIl0gUy7rKCIR\nEREREZGkaE+XiIiIiIhIgtR0iYiIiIiIJEhNl4iIiIiISILUdImIiIiIiCRITZeIiIiIiEiCdHFk\nEZEOYGb7AqPAXeFduwM3AIPu/us6z/2uux+Z4NjOAW5x92/UyHwe+IC7/yKpcYiIiKSV9nSJiHSO\n+9z9MHc/DDgAuB/4agPPW5rkoNz9/bUartCRQCbJcYiIiKSVrtMlItIBwj1dI+6+b+S+PwN+DbyS\nYA/YRcBBwHOBO4CTgfOAdwM3u/tiMxsATgH+HHgUONndfcrv+jnw38DR4V2nuvsPzKwXuBjYE3gI\nON3dv29mlwIj4dfXgTuBw8Ox9QFvA84Bfgb8HXBmuOwngA3ufnbT/0AiIiIppj1dIiIdyt0fBX5K\nsNfrZcCj7v5SYH9gHvAadz89zC42s2cCy4Gl7n4Q8E1goMriH3L3w4H3A18I71sPfMLdDwH+Gfiq\nme065XmHAheEy58A3uDuHwbuA14D7AEsc/dDgSXAgWa2W7P/FiIiImmmpktEpLOVgT+4+7XAp8zs\nXUAe+BuCBmeSu28HXg+cZGYfAo6fmom4OHzO/wILzGwBsL+7D4f33wT8BrApz9vm7j8Iv7+TYK9Y\n1L3AH8zseiAHrHX3R2b4mkVERDqKmi4RkQ4VHl5owF1mdgLwReBh4PPAtUw5h8rM/gq4kWAv2JXA\npVMzEY9Hvt8JmDNNJsNTJ2SKNlDlqct398eBxcD/A54N3BgetigiIjJrqekSEelAZrYTcDZwk7tv\nBY4C/tvdP09wWN+R/KlR2mFmOwMvBn7m7v8JfB94LdM3UwAnhb/ntcBPwlkH7zazFeH9LwH2Jtib\n1YjHgZ3N7HDgGuBad/9XgnPRpu4tExERmVU0ZbyISOfoMbPbw+/nAD8gmCwD4BLgCjM7mWCCjOuB\nheFj/wP8kGAP0zvM7C6CPVDXEEy8MZ0lZvZmggkz3hjetwr4tJmdDfwRWOHuj5o11DN9E/gWcAzB\n3rY7zezhcJxXNrIAERGRTqXZC0VE5EnC2QuXuvvP2zsSERGR2UGHF4qIiIiIiCRIe7pEREREREQS\npD1dIiIiIiIiCVLTJSIiIiIikiA1XSIiIiIiIglS0yUiIiIiIpIgNV0iIiIiIiIJUtMlIiIiIiKS\noP8PjsOK1r54jxsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x144 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mglearn.plots.plot_cross_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均精度 0.957863408521\n"
     ]
    }
   ],
   "source": [
    "#交叉验证\n",
    "from sklearn.model_selection import cross_val_score #交叉验证\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import KFold \n",
    "kfold = KFold(n_splits=10) #可以通过kfold(交叉验证分离器)来控制cv参数\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "rf = RandomForestClassifier(n_estimators=100)\n",
    "scores = cross_val_score(rf,cancer.data,cancer.target,cv=kfold) #cv是折数参数, 默认值为3\n",
    "print(\"平均精度\",scores.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 网络搜索 -- 通过调参提升模型泛化能力"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 简单的网络搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳分数: 0.982456140351\n",
      "最佳参数组: {'n_estimators': 60, 'max_depth': 7}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import  train_test_split\n",
    "from sklearn.datasets import  load_breast_cancer\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "x_train,x_test,y_train,y_test = train_test_split(cancer.data,cancer.target,random_state=0,test_size=0.3)\n",
    "\n",
    "#对两种参数组合都训练一个rf模型\n",
    "best_score = 0\n",
    "for (max_depth,n_estimators) in [(x,y) for x in range(2,10) for y in range(20,100,10)]:\n",
    "    rf = RandomForestClassifier(n_estimators=n_estimators,max_depth=max_depth)\n",
    "    rf.fit(x_train,y_train)\n",
    "    #在测试集上评估rf\n",
    "    score = rf.score(x_test,y_test)\n",
    "    #如果我们得到了更高得分数,则保存该分数下的参数\n",
    "    if score > best_score:\n",
    "        best_score = score\n",
    "        best_parameters = {\"n_estimators\":n_estimators,\"max_depth\":max_depth}\n",
    "\n",
    "print(\"最佳分数:\",best_score)\n",
    "print(\"最佳参数组:\",best_parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 带交叉验证的网络搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "最佳分数: 0.964881229623\n",
      "最佳参数组: {'n_estimators': 50, 'max_depth': 6}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier #随机森林分类器\n",
    "from sklearn.model_selection import  train_test_split #数据集合拆分器\n",
    "from sklearn.datasets import  load_breast_cancer #数据集\n",
    "from sklearn.model_selection import cross_val_score #交叉验证\n",
    "from sklearn.model_selection import KFold #k折交叉验证\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "x_train,x_test,y_train,y_test = train_test_split(cancer.data,cancer.target,random_state=0,test_size=0.3)\n",
    "\n",
    "#对两种参数组合都训练一个rf模型\n",
    "best_score = 0\n",
    "for (max_depth,n_estimators) in [(x,y) for x in range(2,10) for y in range(20,100,10)]:\n",
    "    rf = RandomForestClassifier(n_estimators=n_estimators,max_depth=max_depth)\n",
    "    #k折交叉验证\n",
    "    kfold = KFold(n_splits=5)\n",
    "    \n",
    "    scores = cross_val_score(rf,cancer.data,cancer.target,cv=kfold)\n",
    "    score = np.mean(scores)\n",
    "    \n",
    "    #如果我们得到了更高得分数,则保存该分数下的参数\n",
    "    if score > best_score:\n",
    "        best_score = score\n",
    "        best_parameters = {\"n_estimators\":n_estimators,\"max_depth\":max_depth}\n",
    "\n",
    "print(\"最佳分数:\",best_score)\n",
    "print(\"最佳参数组:\",best_parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 估计器 GridSearchCV方法---避免使用for循环,节约计算资源"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95906432748538006"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier #随机森林分类器\n",
    "from sklearn.model_selection import  train_test_split #数据集合拆分器\n",
    "from sklearn.datasets import  load_breast_cancer #数据集\n",
    "from sklearn.model_selection import KFold #k折交叉验证\n",
    "from sklearn.model_selection import GridSearchCV #估计器\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "x_train,x_test,y_train,y_test = train_test_split(cancer.data,cancer.target,random_state=0,test_size=0.3)\n",
    "\n",
    "#构建参数组集\n",
    "param_grid ={\"n_estimators\": [x for x in range(20,100,10)],\"max_depth\":[y for y in range(2,10)]}\n",
    "\n",
    "#构建网络搜索分类器\n",
    "rf = RandomForestClassifier()\n",
    "kfold = KFold(n_splits=5)\n",
    "grid_search = GridSearchCV(rf,param_grid=param_grid, cv=kfold)\n",
    "\n",
    "#进行交叉验证\n",
    "grid_search.fit(x_train,y_train)\n",
    "\n",
    "grid_search.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.03094</td>\n",
       "      <td>0.00189</td>\n",
       "      <td>0.93719</td>\n",
       "      <td>0.96671</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>{'max_depth': 2, 'n_estimators': 20}</td>\n",
       "      <td>56</td>\n",
       "      <td>0.93750</td>\n",
       "      <td>0.96541</td>\n",
       "      <td>...</td>\n",
       "      <td>0.96250</td>\n",
       "      <td>0.96226</td>\n",
       "      <td>0.93671</td>\n",
       "      <td>0.96552</td>\n",
       "      <td>0.93671</td>\n",
       "      <td>0.96865</td>\n",
       "      <td>0.00372</td>\n",
       "      <td>0.00056</td>\n",
       "      <td>0.01586</td>\n",
       "      <td>0.00321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.04721</td>\n",
       "      <td>0.00223</td>\n",
       "      <td>0.93970</td>\n",
       "      <td>0.96860</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>{'max_depth': 2, 'n_estimators': 30}</td>\n",
       "      <td>52</td>\n",
       "      <td>0.97500</td>\n",
       "      <td>0.97170</td>\n",
       "      <td>...</td>\n",
       "      <td>0.92500</td>\n",
       "      <td>0.97170</td>\n",
       "      <td>0.93671</td>\n",
       "      <td>0.96552</td>\n",
       "      <td>0.96203</td>\n",
       "      <td>0.96552</td>\n",
       "      <td>0.01072</td>\n",
       "      <td>0.00031</td>\n",
       "      <td>0.02667</td>\n",
       "      <td>0.00276</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "0        0.03094          0.00189          0.93719           0.96671   \n",
       "1        0.04721          0.00223          0.93970           0.96860   \n",
       "\n",
       "  param_max_depth param_n_estimators                                params  \\\n",
       "0               2                 20  {'max_depth': 2, 'n_estimators': 20}   \n",
       "1               2                 30  {'max_depth': 2, 'n_estimators': 30}   \n",
       "\n",
       "   rank_test_score  split0_test_score  split0_train_score       ...         \\\n",
       "0               56            0.93750             0.96541       ...          \n",
       "1               52            0.97500             0.97170       ...          \n",
       "\n",
       "   split2_test_score  split2_train_score  split3_test_score  \\\n",
       "0            0.96250             0.96226            0.93671   \n",
       "1            0.92500             0.97170            0.93671   \n",
       "\n",
       "   split3_train_score  split4_test_score  split4_train_score  std_fit_time  \\\n",
       "0             0.96552            0.93671             0.96865       0.00372   \n",
       "1             0.96552            0.96203             0.96552       0.01072   \n",
       "\n",
       "   std_score_time  std_test_score  std_train_score  \n",
       "0         0.00056         0.01586          0.00321  \n",
       "1         0.00031         0.02667          0.00276  \n",
       "\n",
       "[2 rows x 22 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#可视化参数组合结果\n",
    "result = pd.DataFrame(grid_search.cv_results_)\n",
    "result.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PolyCollection at 0x1205a30f0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAJMCAYAAAABj1m5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzs3Xt4FfWh7//3gtxZBhASSAghF8MQ\nmovGIBzFC7WKhaLwc6uIqKCpZRdo5QBVtJRms91cBGktIFWKe9NYlKeboogo0GIjyiWh3BKaIYGE\nhEAgNAiEkGTlcv6IrM2avdWc55dhkpzP63l86vglmc98OzN81lwSV3NzMyIiIiLyX7o4HUBERESk\nvVFBEhEREbFQQRIRERGxUEESERERsVBBEhEREbFQQRIRERGx8HM6wFXp6en6eQNfGTJkiNMR2pUz\nZ844HaFdKS0tdTpCuxIdHe10hHble9/7ntMRpB1bs2aN0xHaldzcXNfXjekKkoiIiIiFCpKIiIiI\nhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiF\nCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUK\nkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqS\niIiIiIUKkoiIiIiFCpKIiIiIhZ/TAa4nl8vFiy++SEJCAh6Ph/nz53Py5Env+NNPP83999/P5cuX\nWbt2LTt37iQoKIg5c+YQGRmJv78/r776Kvn5+Q5uRdtxuVyMHz+eqKgoGhoayMrKorKy0jt+//33\nk56eTm1tLVu3biUvL4+QkBAyMzM5deoUAAcOHGDHjh1ObUKbcblcPPfcc8TExODxeFi5ciUVFRXe\n8XHjxjF8+HBqamrYuHEj+/btw+12s3z5ckpLSwHYs2cPmzdvdmoT2pSOFV/aP3w1NTXx3nvvUV5e\njp+fHxMmTCA8PNw7vnXrVnJzcwkKCuK+++4jOTmZy5cvk5mZSWRkJACpqamMGDHCqU1oU5oPX53l\n/GFbQTIMIxB4G4gDLgJTTdMstGt9rXHPPfcQEBDAM888Q1JSEjNmzGDmzJkAxMfHM3LkSCZNmgTA\nmjVryMnJ4amnnuLYsWPMmzePm266iYEDBzr+f1pbSU1N9e6IsbGxPPzww6xatQqAyMhIhgwZwqJF\niwCYPXs2pmkSHR1NTk4O69evdzJ6m7vtttvw9/dnzpw5DBw4kEmTJrFw4UIAoqOjufPOO3nhhRcA\nWLBgAYcPHyYuLo6dO3eyevVqJ6PbQseKL+0fvg4dOoTH42HWrFkUFxezYcMGpkyZAkB5eTm5ubnM\nnj0bgKVLl2IYBmVlZaSnp/Poo486Gd0Wmg9fneX8Yectth8C1aZpDgOmA8ttXFer3HzzzezatQuA\nvLw8EhMTvWOxsbHs27eP+vp66uvrKS0tJSEhgWHDhuHxePjNb35DRkaG9+s7g/j4eI4cOQJAcXEx\nAwYM8I717duXo0eP0tDQQENDA2fPniUqKoro6Giio6OZMWMGGRkZhIaGOhW/TSUmJrJ//34Ajh49\nSnx8vHcsKiqKvLw8PB4PHo+H06dPExMTQ3x8PHFxccyfP59Zs2bRs2dPp+K3OR0rvrR/+Dp27BiD\nBw8GWvaHq1fJACoqKkhISMDf3x9/f3/CwsIoLy+ntLSUsrIyli1bxurVq7lw4YJT8duc5sNXZzl/\n2FmQBgNbAEzTNIHEb/7j9uvWrRvV1dXe5aamJrp27QpAUVERaWlphISE0L17d1JSUggODqZHjx7c\ncMMNTJ8+nc8++4znn3/eqfhtLjg4mCtXrniXm5qa6NKlZZc4deoUCQkJBAYG0q1bN+Li4ggICKCi\nooIPP/yQZcuWcfDgQR577DGn4repkJAQampqvMvXzsWJEycYPHgwQUFBuN1uDMMgMDCQkydP8u67\n7zJ37lz27t1LRkaGU/HbnI4VX9o/fNXW1hIcHOxd7tKlC42NjQD069ePoqIiamtrqa6u5vjx49TV\n1dGnTx9Gjx7NjBkzSElJ6VRXoTUfvjrL+cPOZ5AOAD8wDGMjMBToZxhGV9M0G21c5ze6fPkyISEh\n3mWXy+XdiUtKSli/fj2vv/46ZWVl5Ofn8+WXX3LhwgWys7MByM7O5umnn3Ykux2uXLlCYGCgd9nl\nctHU1AS0fOr59NNPmTZtGpWVlZSUlFBdXU1JSQn19fVAy/NHY8aMcSR7W6upqflvJ7irc1FeXs6W\nLVuYO3cup0+fprCwkEuXLlFYWOidi927dzN+/HhHsttBx4ov7R++goKCqK2t9S43Nzd7/wLs27cv\nd999NytWrCAsLIyYmBjcbjcxMTEEBAQALVcYOsvzWKD5sOos5w87ryCtoeXZox3AGGCfk+UI4ODB\ng9xxxx0AJCUlUVRU5B3r0aMHPXr0ICMjgyVLltCnTx+OHTvGgQMHvF+TlpbG8ePHHcluh+PHj5OU\nlAS0XPa8+uA1gNvtxu12s3TpUtavX0/Pnj05deoUEydO5JZbbgFg0KBBPpeSO7KCggLS0tIAGDhw\nICdOnPCOhYaGEhoayssvv8yaNWvo1asXpaWlTJ06lWHDhgGQkpLCsWPHHMluBx0rvrR/+IqLi/M+\nH1JcXOx90Bjg0qVLVFdXM3PmTB555BHOnz9PZGQk77zzjvc2pWma9O/f35HsdtB8+Oos5w9Xc3Oz\nLd/YMIz/BUSapvmfhmGkA7NM0/zaj1Dp6en2BLnG1Sfrb7rpJlwuF5mZmQwfPpyysjKys7N56aWX\nMAyDhoYGli9fzv79+wkNDeXnP/85vXv3pqGhgXnz5nH69Glbcw4ZMsTW73/V1bfY+vXrh8vlYu3a\ntSQlJVFZWcmhQ4eYMGEC/fv3p7GxkY0bN1JUVESvXr148skncblc1NXVkZWVxcWLF23NeebMGVu/\nP/zXW0oDBgzA5XKxfPly0tLSqKioICcnhylTphAXF+d92+/IkSOEh4czdepU71ysXLmS8+fP2571\nepTSjnKsQMtD0nbrSPvH9773PdvXce1bWwATJ04kPz+fsLAwkpOTWbduHWVlZfj5+fHggw+SkJDA\nuXPnyMrKAiAgIIAnnniC7t272571euhI87FmzRrb19GRzh+5ubmur90OGwtSb+BdoBvwJfCsaZqn\nvu7PX4+C1FFcr4LUUVyPgtSRdJardm3lehSkjuR6FCTpuK5HQepIvqkg2fYMkmma5wAdqSIiItLh\n6Cdpi4iIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhY\nqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFio\nIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKgg\niYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWLiam5udzgCAy+VqH0Gk3Vmx\nYoXTEaQd2759u9MR2pVdu3Y5HaFdqaiocDpCuzJu3DinI7QrGzZscH3dmK4giYiIiFioIImIiIhY\nqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFio\nIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKgg\niYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJ\niIiIWKggiYiIiFj4OR3genK5XKxcuZLU1FTq6urIyMjg2LFj3vGf/exnPP7441y8eJHFixezefNm\nQkJCeOONN4iNjSUgIIDp06eTk5Pj4Fa0Dc2Fr6amJt577z3Ky8vx8/NjwoQJhIeHe8e3bt1Kbm4u\nQUFB3HfffSQnJ3P58mUyMzOJjIwEIDU1lREjRji1CW1K8+HL5XLx3HPPERMTg8fjYeXKlVRUVHjH\nx40bx/Dhw6mpqWHjxo3s27cPt9vN8uXLKS0tBWDPnj1s3rzZqU1oUy6Xi4ULFzJ48GDq6+uZOXMm\nJSUl3vGpU6cyduxYqqurWbFiBdu3byc4OJhFixYRHR2Nv78/L7/8MgcOHHBuI9qQzqe+OsvxYltB\nMgzDH/gPIAZoBH5ommaBXetrjbFjxxIUFMTtt9/O0KFDWbp0KWPHjgUgKSmJCRMmMHToUAC++OIL\n/vKXvzB79mzy8vJ4+umnSU5OJjU1tVPsxJoLX4cOHcLj8TBr1iyKi4vZsGEDU6ZMAaC8vJzc3Fxm\nz54NwNKlSzEMg7KyMtLT03n00UedjG4LzYev2267DX9/f+bMmcPAgQOZNGkSCxcuBCA6Opo777yT\nF154AYAFCxZw+PBh4uLi2LlzJ6tXr3Yyui2+//3vExgYyJgxY0hLS2PevHlMnjwZgEGDBjFu3DhG\njx4NwKZNm/j888/58Y9/TEFBAT/5yU9ITEzkO9/5TqcpSDqf+uosx4udt9hGAX6mad4O/Avwio3r\napXhw4fz8ccfAy3tND093TuWmJjIp59+Sl1dHXV1dRQWFpKSksLIkSOpr6/n448/Zu7cuXzyySdO\nxW9Tmgtfx44dY/DgwQDExsZ6P8UAVFRUkJCQgL+/P/7+/oSFhVFeXk5paSllZWUsW7aM1atXc+HC\nBafitznNh6/ExET2798PwNGjR4mPj/eORUVFkZeXh8fjwePxcPr0aWJiYoiPjycuLo758+cza9Ys\nevbs6VT8NnfbbbexY8cOAP72t7+RmprqHUtISGDXrl3e88fx48dJTEzknnvuwePxsG7dOmbMmOH9\n+s5A51NfneV4sbMgHQX8DMPoAoQCHhvX1SqhoaE+J+3Gxka6du0KwOHDh7nrrrtwu93ceOON3H77\n7XTr1o3evXvTs2dPHnjgATZt2sSSJUucit+mNBe+amtrCQ4O9i536dKFxsZGAPr160dRURG1tbVU\nV1dz/Phx6urq6NOnD6NHj2bGjBmkpKSwfv16p+K3Oc2Hr5CQEGpqarzLTU1NdOnScvo8ceIEgwcP\nJigoCLfbjWEYBAYGcvLkSd59913mzp3L3r17ycjIcCp+m3O73Vy6dMm73NTU5D1/FBQUMHToULp1\n60bPnj1JT08nJCSEG2+8ke7du/P444+zbds25s2b51T8Nqfzqa/OcrzY+QxSNS231wqA3sAPbFxX\nq1y8eJEbbrjBu3ztSb+goIDly5ezZcsWioqK2LNnD+fOneMf//gHH3zwAdByqfjFF190JHtb01z4\nCgoKora21rvc3NzsPcH17duXu+++mxUrVhAWFkZMTAxut5uYmBgCAgIAuPnmmx2/X96WNB++ampq\n/lthbGpqAlpuOW7ZsoW5c+dy+vRpCgsLuXTpEoWFhdTX1wOwe/duxo8f70h2O1RXV9OtWzfvssvl\n8p4/CgsLefvtt/nDH/5AcXEx+/fvp6qqivPnz7N161ag5Rm2adOmOZLdDjqf+uosx4udV5BmAJ+Y\npjkQSAX+wzCMIBvX960+//xzRo0aBcDQoUM5fPiwd6x379707t2bO++8k5/+9Kf079+fvLw8du7c\n6f2au+66i/z8fEeytzXNha+4uDjv9hQXF3sfNAa4dOkS1dXVzJw5k0ceeYTz588TGRnJO++8472M\nbJom/fv3dyS7HTQfvgoKCkhLSwNg4MCBnDhxwjsWGhpKaGgoL7/8MmvWrKFXr16UlpYydepUhg0b\nBkBKSorPQ7sdXU5ODvfeey8AaWlpFBT81+OlvXr14sYbb+Shhx5i7ty5REZGUlBQwN69e71fM2zY\nMEzTdCS7HXQ+9dVZjhdXc3OzLd/YMIy5gMc0zYWGYXQD8oHvmKZ5+X8M4nLZE8R3HaxcuZKUlBRc\nLheTJ09m1KhRFBUVsWnTJlatWkVaWhr19fXMmTOHzz77jJ49e7J69WoiIiLweDw89dRTPv9nd1Qd\naS5WrFhh+zqufWsLYOLEieTn5xMWFkZycjLr1q2jrKwMPz8/HnzwQRISEjh37hxZWVkABAQE8MQT\nT9C9e3fbs14PHWk+tm/fbvs6rr6VM2DAAFwuF8uXLyctLY2KigpycnKYMmUKcXFxNDQ0kJWVxZEj\nRwgPD2fq1Km4XC7q6upYuXIl58+ftz3rrl27bF/H1bfYEhMTcblczJgxg3vvvZfi4mK2bt3K4sWL\nSU5Opr6+ngULFrB792569OjB0qVLCQ8Pp6GhgenTp3Py5Enbs1779pRdOtL5dNy4cbavoyMdLxs2\nbHB97XbYWJDcwBogAggAfm2a5h++Nsh1KEjSMV2PgiQd1/UoSB3J9ShIHcn1KEgdyfUoSB3JNxUk\n255BMk2zGuh87/uKiIhIp6efpC0iIiJioYIkIiIiYqGCJCIiImKhgiQiIiJioYIkIiIiYqGCJCIi\nImKhgiQiIiJioYIkIiIiYqGCJCIiImKhgiQiIiJioYIkIiIiYqGCJCIiImKhgiQiIiJioYIkIiIi\nYqGCJCIiImKhgiQiIiJioYIkIiIiYqGCJCIiImKhgiQiIiJioYIkIiIiYqGCJCIiImKhgiQiIiJi\noYIkIiIiYqGCJCIiImKhgiQiIiJioYIkIiIiYqGCJCIiImKhgiQiIiJioYIkIiIiYqGCJCIiImLh\n53SAq1asWOF0hHbjzJkzTkdoVw4fPux0hHYlOTnZ6Qjtyq5du5yO0K7069fP6QjtiubDV58+fZyO\n0GHoCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiI\niIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiI\nhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhQqSiIiIiIUKkoiIiIiF\nCpKIiIiIhQqSiIiIiIUKkoiIiIiFCpKIiIiIhZ/TAa6npqYm3nvvPcrLy/Hz82PChAmEh4d7x7du\n3Upubi5BQUHcd999JCcnc/nyZTIzM4mMjAQgNTWVESNGOLUJbcrlcjF69Gj69u1LQ0MDH3zwAVVV\nVd7x4cOHk5SURF1dHZ9//jlHjx4lODiY6dOnc/bsWQAKCgrYvXu3U5vQZlwuF+PHjycqKoqGhgay\nsrKorKz0jt9///2kp6dTW1vL1q1bycvLIyQkhMzMTE6dOgXAgQMH2LFjh1Ob0KZ0rPhyuVwsXLiQ\nwYMHU19fz8yZMykpKfGOT506lbFjx1JdXc2KFSvYvn07wcHBLFq0iOjoaPz9/Xn55Zc5cOCAcxvR\nhlwuFy+++CIJCQl4PB7mz5/PyZMnveNPP/00999/P5cvX2bt2rXs3LmToKAg5syZQ2RkJP7+/rz6\n6qvk5+c7uBVtR/Phq7OcT20rSIZhTAImfbUYBNwM9DVN80u71vltDh06hMfjYdasWRQXF7Nhwwam\nTJkCQHl5Obm5ucyePRuApUuXYhgGZWVlpKen8+ijjzoV2zaDBg3Cz8+P1atXExUVxciRI1m3bh0A\n4eHhJCcn89ZbbwHw7LPPUlxcTEREBHl5eXz00UdORm9zqamp3pNUbGwsDz/8MKtWrQIgMjKSIUOG\nsGjRIgBmz56NaZpER0eTk5PD+vXrnYxuCx0rvr7//e8TGBjImDFjSEtLY968eUyePBloOY7GjRvH\n6NGjAdi0aROff/45P/7xjykoKOAnP/kJiYmJfOc73+k0Bemee+4hICCAZ555hqSkJGbMmMHMmTMB\niI+PZ+TIkUyaNAmANWvWkJOTw1NPPcWxY8eYN28eN910EwMHDuw0hUDz4auznE9tu8Vmmua/m6Z5\nj2ma9wD7gJ84WY4Ajh07xuDBgwGIjY2ltLTUO1ZRUUFCQgL+/v74+/sTFhZGeXk5paWllJWVsWzZ\nMlavXs2FCxecit/moqOjKSoqAuDkyZPeT/4AYWFhlJSU0NDQQENDA1VVVfTp04fIyEgiIiKYPHky\njz76KG6326n4bSo+Pp4jR44AUFxczIABA7xjffv25ejRo965OHv2LFFRUURHRxMdHc2MGTPIyMgg\nNDTUqfhtTseKr9tuu837afZvf/sbqamp3rGEhAR27dpFXV0ddXV1HD9+nMTERO655x48Hg/r1q1j\nxowZjn8abks333wzu3btAiAvL4/ExETvWGxsLPv27aO+vp76+npKS0tJSEhg2LBheDwefvOb35CR\nkeH9+s5A8+Grs5xPbX8GyTCMdOA7pmm+afe6vk1tbS3BwcHe5S5dutDY2AhAv379KCoqora2lurq\nao4fP05dXR19+vRh9OjRzJgxg5SUlHbVbv//CgwMpLa21rvc1NREly4tu8SZM2cYMGAAAQEBBAcH\n079/f/z9/amsrGTHjh28/fbb/P3vf2fUqFFOxW9TwcHBXLlyxbt87VycOnWKhIQEAgMD6datG3Fx\ncQQEBFBRUcGHH37IsmXLOHjwII899phT8ducjhVfbrebS5cueZebmpro2rUr0HKbeejQoXTr1o2e\nPXuSnp5OSEgIN954I927d+fxxx9n27ZtzJs3z6n4ba5bt25UV1d7l6+dj6KiItLS0ggJCaF79+6k\npKQQHBxMjx49uOGGG5g+fTqfffYZzz//vFPx25zmw1dnOZ9ej2eQXgIyr8N6vlVQUJBPIWhubvbu\nxH379uXuu+9mxYoVhIWFERMTg9vtJiYmhoCAAKDlU8LmzZsdyW6Huro6AgMDvcsul4umpiYAzp07\nx969e5k4cSJVVVWcPHmSmpoaysvL8Xg8QMtfDN/97ncdyd7Wrly58rVzUVFRwaeffsq0adOorKyk\npKSE6upqSkpKqK+vB1rul48ZM8aR7HbQseKrurqabt26eZddLpe3MBYWFvL222/zhz/8geLiYvbv\n309VVRXnz59n69atQMszW9OmTXMkux0uX75MSEiId/na+SgpKWH9+vW8/vrrlJWVkZ+fz5dffsmF\nCxfIzs4GIDs7m6efftqR7HbQfPjqLOdTW68gGYbRAxhkmma7uLYcFxfnvcdbXFzsc0vp0qVLVFdX\nM3PmTB555BHOnz9PZGQk77zzDvv37wfANE369+/vSHY7XL3UCxAVFeV98BogJCSEkJAQ1qxZw5Yt\nW+jevTtnz57loYce8rn1cvWBuo7u+PHjJCUlAf99u9xuN263m6VLl7J+/Xp69uzJqVOnmDhxIrfc\ncgvQ8hzKtbehOjodK75ycnK49957AUhLS6OgoMA71qtXL2688UYeeugh5s6dS2RkJAUFBezdu9f7\nNcOGDcM0TUey2+HgwYPccccdACQlJXlv1QP06NGDHj16kJGRwZIlS+jTpw/Hjh3jwIED3q9JS0vj\n+PHjjmS3g+bDV2c5n7qam5tt++aGYTwIfM80zZ98259duXKlfUG+cu2bOQATJ04kPz+fsLAwkpOT\nWbduHWVlZfj5+fHggw+SkJDAuXPnyMrKAiAgIIAnnniC7t2725rzzJkztn7/q66+xdanTx9cLhcb\nN24kISGBqqoqTNNkzJgxRERE0NjYyPbt2zlx4gQ9evRg7NixAHg8Ht5//32fS8t2uLa42eXqWxf9\n+vXD5XKxdu1akpKSqKys5NChQ0yYMIH+/fvT2NjIxo0bKSoqolevXjz55JO4XC7q6urIysri4sWL\ntmdNTk62fR0d5VgBmD9/vu3ruPoWW2JiIi6XixkzZnDvvfdSXFzM1q1bWbx4McnJydTX17NgwQJ2\n795Njx49WLp0KeHh4TQ0NDB9+nSfN5vs0q9fP9vXcfWtrZtuugmXy0VmZibDhw+nrKyM7OxsXnrp\nJQzDoKGhgeXLl7N//35CQ0P5+c9/Tu/evWloaGDevHmcPn3a9qzXQ0eajyFDhti+jo50Pn3jjTdc\nX7sdNhek2YDHNM1ffdufvR4FqaO4XgWpo7geBakjuR4FqSO5HgWpI7keBUk6rutRkDqSbypItj6D\nZJrmq3Z+fxERERE76Cdpi4iIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKgg\niYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJ\niIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImI\niIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWKggiYiIiFioIImIiIhYqCCJiIiIWLiam5ud\nzgBAYWFh+wjSDgwcONDpCO3KlClTnI7QroSHhzsdoV05e/as0xFEOowzZ844HaFd2bBhg+vrxnQF\nSURERMRCBUlERETEQgVJRERExEIFSURERMRCBUlERETEQgVJRERExEIFSURERMRCBUlERETEQgVJ\nRERExEIFSURERMRCBUlERETEQgVJRERExEIFSURERMRCBUlERETEQgVJRERExEIFSURERMRCBUlE\nRETEQgVJRERExEIFSURERMRCBUlERETEQgVJRERExEIFSURERMRCBUlERETEQgVJRERExEIFSURE\nRMRCBUlERETEQgVJRERExEIFSURERMTCz+kA11NTUxO//OUvKSgoICAggFdeeYUBAwZ4x998800+\n/PBD3G43P/zhDxkxYoR3LCcnh5kzZ5Kdne1E9DbncrlYuXIlqamp1NXVkZGRwbFjx7zjP/vZz3j8\n8ce5ePEiixcvZvPmzYSEhPDGG28QGxtLQEAA06dPJycnx8GtaDsul4vx48cTFRVFQ0MDWVlZVFZW\nesfvv/9+0tPTqa2tZevWreTl5RESEkJmZianTp0C4MCBA+zYscOpTWhTLpeL0aNH07dvXxoaGvjg\ngw+oqqryjg8fPpykpCTq6uoidZJPAAAgAElEQVT4/PPPOXr0KMHBwUyfPp2zZ88CUFBQwO7du53a\nhDal/cOX5sOX5sOXy+XiueeeIyYmBo/Hw8qVK6moqPCOjxs3juHDh1NTU8PGjRvZt28fbreb5cuX\nU1paCsCePXvYvHmzU5sA2FyQDMOYAzwIBAArTdP8nZ3r+zbbtm2jrq6O9evXc+DAARYuXMgbb7wB\ngGmabNq0iT/+8Y8APPbYYwwbNozg4GBOnz7NmjVraGhocDJ+mxo7dixBQUHcfvvtDB06lKVLlzJ2\n7FgAkpKSmDBhAkOHDgXgiy++4C9/+QuzZ88mLy+Pp59+muTkZFJTUztNQUpNTcXf359XX32V2NhY\nHn74YVatWgVAZGQkQ4YMYdGiRQDMnj0b0zSJjo4mJyeH9evXOxndFoMGDcLPz4/Vq1cTFRXFyJEj\nWbduHQDh4eEkJyfz1ltvAfDss89SXFxMREQEeXl5fPTRR05Gt4X2D1+aD1+aD1+33XYb/v7+zJkz\nh4EDBzJp0iQWLlwIQHR0NHfeeScvvPACAAsWLODw4cPExcWxc+dOVq9e7WR0H7bdYjMM4x7gduAO\n4G6gv13raq19+/Zx5513AnDzzTdz+PBh79ixY8cYOnQogYGBBAYGMmDAAEzTpK6ujl/84hfMmzfP\nqdi2GD58OB9//DHQ0tTT09O9Y4mJiXz66afU1dVRV1dHYWEhKSkpjBw5kvr6ej7++GPmzp3LJ598\n4lT8NhcfH8+RI0cAKC4u9rmy2LdvX44ePUpDQwMNDQ2cPXuWqKgooqOjiY6OZsaMGWRkZBAaGupU\n/DYXHR1NUVERACdPniQyMtI7FhYWRklJiXc+qqqq6NOnD5GRkURERDB58mQeffRR3G63U/HbnPYP\nX5oPX5oPX4mJiezfvx+Ao0ePEh8f7x2LiooiLy8Pj8eDx+Ph9OnTxMTEEB8fT1xcHPPnz2fWrFn0\n7NnTqfhedj6DNBI4DPwJ2AR8aOO6WqW6upobbrjBu9y1a1fvVaGBAweSk5NDdXU158+fZ//+/dTU\n1JCZmcmzzz5L3759nYpti9DQUC5cuOBdbmxspGvXrgAcPnyYu+66C7fbzY033sjtt99Ot27d6N27\nNz179uSBBx5g06ZNLFmyxKn4bS44OJgrV654l5uamujSpeXwOHXqFAkJCQQGBtKtWzfi4uIICAig\noqKCDz/8kGXLlnHw4EEee+wxp+K3ucDAQGpra73L187HmTNnGDBgAAEBAQQHB9O/f3/8/f2prKxk\nx44dvP322/z9739n1KhRTsVvc9o/fGk+fGk+fIWEhFBTU+NdvnY+Tpw4weDBgwkKCsLtdmMYBoGB\ngZw8eZJ3332XuXPnsnfvXjIyMpyK72XnLbbewADgB0As8IFhGINM02y2cZ3fyO12c/nyZe9yU1MT\nfn4tU3DTTTcxceJEMjIyGDBgAKmpqfj5+bFv3z5KS0tZvnw5Fy5c4Pnnn+dXv/qVU5vQZi5evOhT\nFrt06UJjYyPQ8uzI8uXL2bJlC0VFRezZs4dz587xj3/8gw8++ACATZs28eKLLzqS3Q5XrlwhMDDQ\nu+xyuWhqagKgoqKCTz/9lGnTplFZWUlJSQnV1dWUlJRQX18PtDw/MGbMGEey26Guru5r5+PcuXPs\n3buXiRMnUlVVxcmTJ6mpqaG8vByPxwO07EPf/e53HcluB+0fvjQfvjQfvmpqaggODvYud+nSxTsf\n5eXlbNmyhblz53L69GkKCwu5dOkShYWF3vnYvXs348ePdyT7tey8gvQP4BPTNOtN0zSBWiDMxvV9\nq1tvvZW//vWvQMsOOXDgQO9YVVUV58+f59133+XnP/85p0+f5tZbb+WTTz4hKyuLrKwsunfv3inK\nEcDnn3/u/YQ/dOhQn9uNvXv3pnfv3tx555389Kc/pX///uTl5bFz507v19x1113k5+c7kt0Ox48f\nJykpCYDY2Fjvg5PQUqzdbjdLly5l/fr19OzZk1OnTjFx4kRuueUWoOWZnasPF3YGpaWlJCQkAC2X\nxK8+eA0tnw5DQkJYs2YNW7ZsoXv37pw9e5aHHnqIwYMHA/99Djs67R++NB++NB++CgoKSEtLA1ru\nzpw4ccI7FhoaSmhoKC+//DJr1qyhV69elJaWMnXqVIYNGwZASkqKz0tDTnE1N9tzQccwjB8APwXu\nByKAbMAwTbPxf/rzhYWFtl9ZuvoWm2maNDc3s2DBAv76178yYMAAvvvd7/KLX/yC/Px8AgICmDlz\nJkOGDPH5+ttvv50vvvjC7pg+xc0uV99iS0lJweVyMXnyZEaNGkVRURGbNm1i1apVpKWlUV9fz5w5\nc/jss8/o2bMnq1evJiIiAo/Hw1NPPeWz49tlypQptq/j6lso/fr1w+VysXbtWpKSkqisrOTQoUNM\nmDCB/v3709jYyMaNGykqKqJXr148+eSTuFwu6urqyMrK4uLFi7ZnDQ8Pt30dV99i69OnDy6Xi40b\nN5KQkEBVVRWmaTJmzBgiIiJobGxk+/btnDhxgh49engf9Pd4PLz//vtUV1fbnvXa8maXjrR/XA+a\nD18daT7OnDlj+zquvsU2YMAAXC4Xy5cvJy0tjYqKCnJycpgyZQpxcXHeN/6OHDlCeHg4U6dO9c7H\nypUrOX/+vO1ZN2zY4Pra7bCrIAEYhrEYGEHLlaqXTNP82qd6r0dB6iiuR0HqSK5HQepIrkdB6kiu\nR0ES6SyuR0HqSL6pINn6mr9pmj+z8/uLiIiI2EE/SVtERETEQgVJRERExEIFSURERMRCBUlERETE\nolUPaRuGMRqYB/QCXF/902yaZpyN2UREREQc0dq32H5Ny880ygf0Or6IiIh0aq0tSBdM09xsaxIR\nERGRduIbC5JhGHd99a/5hmG8DmwEGq6Om6aZbWM2EREREUd82xWkzGv+vT+QfM1yM9B5fhuliIiI\nyFe+sSCZpjkCwDCM75im6fObSQ3DGGZnMBERERGnfNsttjuArsBqwzCepeXttatftwrQLw0TERGR\nTufbbrHdB9wNRAD/cs1/bwB+a1coERERESd92y22XwIYhvGkaZq/vy6JRERERBzW2tf8sw3DeB8Y\nQcvVo4+AGaZpVtqWTERERMQhrf1VI1nANqAfEAfsA/7DrlAiIiIiTmrtFaRQ0zSXX7O8zDCMSTbk\nEREREXFca68gfWEYxsSrC1/9brb99kQSERERcVZrryD9f8CPDMN4E2gCQgAMw3iKll9a29WmfCIi\nIiLXXasKkmmafewOIiIiItJetKogGYYRAMwCDGA68Dyw0DTNehuziYiIiDiitc8grQDcwK20vOaf\nAKyxK5SIiIiIk1pbkG41TfMlwGOaZg3wFHCzfbFEREREnNPagtT81W225q+We1/z7yIiIiKdSmsL\n0q+A7UBfwzB+BeQCy2xLJSIiIuKg1r7F9nvDMPbR8qtGugJjTNM8ZGsyEREREYd8Y0H66uccXevS\nV/97s2EYN5umudaeWCIiIiLO+bYrSCO++t944CZgM9AIPADkAypIIiIi0ul8Y0EyTXMygGEYO4AU\n0zTPfbXcE9hofzwRERGR66+1D2lHAlXXLF8GIto+joiIiIjzWvu72DYD2wzD2AC4gEeB99oyyLZt\n29ry23Vo48aNczpCu5KZmel0hHZl165dTkdoVz7++GOnI7QrTz1lfXT0/22vvvqq0xGkg2rVFSTT\nNP83sBIYBAwElpimORfAMIy+9sUTERERuf5aewUJ0zT/E/jP/2HoIyCtzRKJiIiIOKy1zyB9E1cb\nfA8RERGRdqMtCpJ+5YiIiIh0Km1RkEREREQ6FRUkEREREQs9gyQiIiJi0aqCZBjG9y3LLsMwZn21\nOL3NU4mIiIg4qLWv+S80DGMMMBOIAv4DOEfLz0PaaVc4ERERESe09hZbOi2/aiQP2E5LMXrQtlQi\nIiIiDmptQYoD7gBM4CJwl2EYIbalEhEREXFQawtSNvDvpmmOouVqkoeWq0kiIiIinU5rn0G61TTN\nkwCmadYBsw3D+KN9sURERESc09qCFGwYxq8BNy2v9XcFYoG77AomIiIi4pTW3mJbB3wJ3AIcAKLR\nLTYRERHppFpbkAJM05wHfAz8DRgF3G1bKhEREREHtbYg1RiGEQgcpeV5pCs2ZhIRERFxVGufQcoC\nNgFPALsMw3gAOGlbKhEREREHtbYgvUfL1aZ/Bj4FhgCf2JRJRERExFGtLUgfAYeAUqDsq39ERERE\nOqXWFiRM03zWziAiIiIi7UVrC9JGwzAygL8ADVf/o2mapbakEhEREXFQawuSG3gROHfNf2um5Xe0\niYiIiHQqrS1IY4Bwvd4vIiIi/y9o7c9BKgF62phDREREpN1o7RWkAOCIYRh5QP3V/2ia5ndtSSUi\nIiLioNYWpFdsTSEiIiLSjrSqIJmm+Ve7g4iIiIi0F63+OUidQVNTE++99x7l5eX4+fkxYcIEwsPD\nveNbt24lNzeXoKAg7rvvPpKTk7l8+TKZmZlERkYCkJqayogRI5zahDblcrl47rnniImJwePxsHLl\nSioqKrzj48aNY/jw4dTU1LBx40b27duH2+1m+fLllJa2/ISHPXv2sHnzZqc2oc00NTXxwgsvkJ+f\nT2BgIK+99hqxsbHe8d/85jf86U9/4oYbbmDq1Kncf//93rFdu3bx4x//mP379zsR3RZNTU389re/\npaSkBD8/P6ZNm0ZERIR3fMOGDWRnZxMSEsK4ceMYMmSIdyw/P5/XXnuN3/3ud05Et4XL5WL8+PFE\nRUXR0NBAVlYWlZWV3vH777+f9PR0amtr2bp1K3l5eYSEhJCZmcmpU6cAOHDgADt27HBqE9pUU1MT\nS5YsobCwkICAAObMmUNUVJR3/Pe//z3bt28nJCSEiRMncscdd3jH9u/fT2ZmJhs3bnQiui10LvXV\nWebD1oJkGMZ+4MJXi8WmaU62c33f5tChQ3g8HmbNmkVxcTEbNmxgypQpAJSXl5Obm8vs2bMBWLp0\nKYZhUFZWRnp6Oo8++qiT0W1x22234e/vz5w5cxg4cCCTJk1i4cKFAERHR3PnnXfywgsvALBgwQIO\nHz5MXFwcO3fuZPXq1U5Gb3Nbtmyhrq6Ojz76iNzcXObNm8fatWsBOHLkCBs2bGDLli0A/OAHP2D4\n8OGEhIRQXl7OG2+8gcfjcTJ+m9uzZw/19fUsWrQI0zR5++23eemllwAoKSkhOzubxYsXA/Diiy+S\nkpJCYGAglZWVvP/++zQ2NjoZv82lpqbi7+/Pq6++SmxsLA8//DCrVq0CIDIykiFDhrBo0SIAZs+e\njWmaREdHk5OTw/r1652Mbovs7Gzq6+t56623yMvL4/XXX/fuD8eOHWPbtm289dZbAPzoRz/i1ltv\nJSgoiDNnzrBu3ToaGhq+6dt3ODqX+uos89Hat9j+rxmGEQRgmuY9X/3jaDmClgN38ODBAMTGxnqb\nKkBFRQUJCQn4+/vj7+9PWFgY5eXllJaWUlZWxrJly1i9ejUXLlz4um/f4SQmJnqvehw9epT4+Hjv\nWFRUFHl5eXg8HjweD6dPnyYmJob4+Hji4uKYP38+s2bNomfPzvFy4549e7xXBtPT0zl48KB3rLCw\nkNtvv52goCCCgoKIjY3lyJEj1NbW8rOf/cz7F2Nn8ve//520tDQADMOgqKjIO3by5EmSkpIICAgg\nICCAiIgISkpKqK+vZ9WqVfzoRz9yKrZt4uPjOXLkCADFxcUMGDDAO9a3b1+OHj1KQ0MDDQ0NnD17\nlqioKKKjo4mOjmbGjBlkZGQQGhrqVPw2d/DgQYYOHQpAUlISBQUF3rGSkhJuueUWAgMDCQwMpH//\n/hQVFVFXV8fixYu9H0I7E51LfXWW+bCtIAGpQIhhGFsNw/iLYRjDbFxXq9TW1hIcHOxd7tKli/eT\nbr9+/SgqKqK2tpbq6mqOHz9OXV0dffr0YfTo0cyYMYOUlJRO9WkwJCSEmpoa73JTUxNdurTsEidO\nnGDw4MEEBQXhdrsxDIPAwEBOnjzJu+++y9y5c9m7dy8ZGRlOxW9Tly5d8vkLrGvXrt5PuYmJieze\nvZvq6mqqqqrIzc2lpqaGOXPm8M///M8+t546i5qaGkJCQrzL1x4rAwYMID8/nytXrnDx4kUKCgqo\nra3lzTffZOzYsfTq1cup2LYJDg7mypX/+jFw1x4rp06dIiEhgcDAQLp160ZcXBwBAQFUVFTw4Ycf\nsmzZMg4ePMhjjz3mVPw2V1NTg9vt9i5fe7zEx8dz4MABLl++zIULFzh8+DC1tbW89tprTJgwgbCw\nMKdi20bnUl+dZT7svMVWAywBVgMJwBbDMAzTNB27thoUFERtba13ubm5ma5duwItnwLvvvtuVqxY\nQVhYGDExMbjdbmJiYggICADg5ptvdvyeaFuqqan5b4WxqakJaLnluGXLFubOncvp06cpLCzk0qVL\nFBYWUl/f8pMedu/ezfjx4x3J3tZuuOEGqqurvctNTU34+bUcHgMHDuSZZ57h8ccfJzY2lrS0NPz8\n/NizZw8lJSUsXbqUL7/8kueee44333zTqU1oUyEhIT6F4NpjpX///owaNYp/+Zd/oW/fvgwcOJCu\nXbty5MgRTp8+zbvvvkt1dTVLlixh1qxZTm1Cm7py5QqBgYHeZZfL5T1WKioq+PTTT5k2bRqVlZWU\nlJRQXV3tvaoGLc8fjRkzxpHsdvif/gK8erzExMTwT//0T8ycOZOoqCgGDx5M165dOXjwICdPnmTN\nmjVcvHiRuXPnMn/+fKc2oU3pXOqrs8yHnVeQjgJZpmk2m6Z5FPgH4OhH7bi4OPLz84GWy+RXH7yG\nlisI1dXVzJw5k0ceeYTz588TGRnJO++8471UaJom/fv3dyS7HQoKCry3UQYOHMiJEye8Y6GhoYSG\nhvLyyy+zZs0aevXqRWlpKVOnTmXYsJaLgSkpKRw7dsyR7G3ttttu489//jMAubm5JCYmesfOnTtH\nVVUVmzZt4l//9V8pLy9n6NChfPHFF/zpT3/iT3/6Ez169Og05Qhg0KBB7Nu3D2jZ76+9pXThwgUu\nXbrEggULyMjI4Ny5cyQmJrJy5UpeeeUVXnnlFdxud6cpRwDHjx8nKSkJaLk9f/XBawC3243b7Wbp\n0qWsX7+enj17curUKSZOnMgtt9wCtMzntbf0O7qUlBR27doFQF5ens8tlPPnz/Pll1+yatUqnn/+\nec6ePUtKSgrvvvsuK1asYMWKFYSGhnaacgQ6l1p1lvmw8wrSM0Ay8GPDMCKBUOC0jev7VqmpqRQU\nFLBkyRIAJk6cyJ///GfCwsJITk7m3LlzLFq0CD8/P8aNG0eXLl146KGHyMrK4rPPPiMgIIAnnnjC\nyU1oU3v27CE1NZV/+7d/w+VysXz5csaMGUNFRQU5OTn06dOHxYsX09DQwNq1a2lqauL3v/89U6dO\n5YEHHqCuro6VK1c6vRltYtSoUfz1r39l9OjRNDc38+tf/5pVq1YRExPDyJEjOXHiBCNHjsTf3595\n8+Z5r6Z0VsOGDePgwYPeBymnT5/O+++/T0REBEOGDKGiooJZs2bh5+fHpEmTOv18HDhwgEGDBjFr\n1ixcLhdr167l3nvvpbKykkOHDtG7d29eeOEFGhsb2bBhA83NzWzcuJEnn3ySu+++m7q6OrKyspze\njDZz9913k5OTw3PPPUdzczMvv/wy69atIyoqiuHDh3Pq1CmeeeYZ/P39mTp1aqffP3Qu9dVZ5sPV\n3Nxsyzc2DCMA+HcgmpZfbPuCaZpffN2fX7lypT1BOqDt27c7HaFdufq2kLS4+sldWnz88cdOR2hX\nnnrqKacjtCuvvvqq0xGkHduwYYPr68Zsu4JkmmY9MMGu7y8iIiJiFzufQRIRERHpkFSQRERERCxU\nkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQ\nRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBE\nRERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkERE\nREQsVJBERERELFSQRERERCxUkEREREQs/JwOcNWZM2ecjtBufO9733M6Qrty4cIFpyNIOxYeHu50\nBGnHnn76aacjtCs6XlpPV5BERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQs\nVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxU\nkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELFSQ\nRERERCxUkEREREQsVJBERERELFSQRERERCxUkEREREQsVJBERERELPycDnA9uVwuRo8eTd++fWlo\naOCDDz6gqqrKOz58+HCSkpKoq6vj888/5+jRowQHBzN9+nTOnj0LQEFBAbt373ZqE9pUU1MT7733\nHuXl5fj5+TFhwgTCw8O941u3biU3N5egoCDuu+8+kpOTuXz5MpmZmURGRgKQmprKiBEjnNqENtPU\n1MQvf/lLCgoKCAgI4JVXXmHAgAHe8TfffJMPP/wQt9vND3/4Q59tzsnJYebMmWRnZzsR3RZNTU38\n9re/paSkBD8/P6ZNm0ZERIR3fMOGDWRnZxMSEsK4ceMYMmSIdyw/P5/XXnuN3/3ud05Et4XOHb6a\nmppYsmQJhYWFBAQEMGfOHKKiorzjv//979m+fTshISFMnDiRO+64wzu2f/9+MjMz2bhxoxPRbaHj\nxVdn2T9sLUiGYYQD+4D7TNMssHNdrTFo0CD8/PxYvXo1UVFRjBw5knXr1gEQHh5OcnIyb731FgDP\nPvssxcXFREREkJeXx0cffeRkdFscOnQIj8fDrFmzKC4uZsOGDUyZMgWA8vJycnNzmT17NgBLly7F\nMAzKyspIT0/n0UcfdTJ6m9u2bRt1dXWsX7+eAwcOsHDhQt544w0ATNNk06ZN/PGPfwTgscceY9iw\nYQQHB3P69GnWrFlDQ0ODk/Hb3J49e6ivr2fRokWYpsnbb7/NSy+9BEBJSQnZ2dksXrwYgBdffJGU\nlBQCAwOprKzk/fffp7Gx0cn4bU7nDl/Z2dnU19fz1ltvkZeXx+uvv+7dH44dO8a2bdu88/GjH/2I\nW2+9laCgIM6cOcO6det0vHTy46Wz7B+23WIzDMMf+C1wxa51/N+Kjo6mqKgIgJMnT3qvggCEhYVR\nUlJCQ0MDDQ0NVFVV0adPHyIjI4mIiGDy5Mn8n/buPzqq6t77+DvkJ7kBHq8NBJo0wUC2YAhgxACC\naLw8aqAiXgpLyFUERSy1hYWWxyogWhc/RPAXimKDrbRQ6rIobbmAwhWNIgFJTITsSgRDgoCYEDKJ\nCT9mnj8mTDPnauvtzXhC8nmt5ZIze5jzPZt99nzmnDNzxo8fT1xcnFvlt7iysjL69u0LQM+ePSkv\nLw+0HT16lN69exMZGUlkZCTx8fFUVlZSXl7O4cOHWb58OS+99BI1NTVuld+i9uzZw/DhwwEYMGAA\nxcXFgbaysjKysrKIjo4mOjqa5ORkrLU0NjYyb9485s+f71bZIbN//34uv/xyAIwxgf0G/PtOeno6\nUVFRREVF0b17dw4dOsTp06dZuXIld999t1tlh4zmjmBFRUVkZWUBkJ6eTmnp3z7/Hjp0iIEDBwb2\nl6SkJA4cOEBjYyNLliwJfOhqS7S/BGsr4yOU1yAtBVYCR0K4jv+R6OhoGhoaAster5cOHfxdcOzY\nMZKTk4mKiqJjx44kJSURGRnJF198wfbt21m9ejX79+8nJyfHrfJbXENDAx07dgwsd+jQIfBJ5vvf\n/z4HDhygoaEBj8fDp59+SmNjI926dWPUqFHMmjWLjIwM1q9f71b5Lcrj8dCpU6fAcnh4eOBTTFpa\nGgUFBXg8Hqqrq9m7dy/19fUsWLCAqVOnkpCQ4FbZIVNfX09sbGxgufnYSE5O5uOPP+arr77i1KlT\nlJaW0tDQwIsvvsjNN9/MxRdf7FbZIaO5I1h9fX1Q4Gu+v6SmplJYWEhdXR01NTUUFxfT0NDAsmXL\nmDhxIvHx8W6VHTLaX4K1lfERklNsxpjJwBfW2s3GmAdCsY5/RmNjI9HR0YHlsLAwvF4vACdOnGDX\nrl3k5uZSVVVFRUUF9fX1VFZWcubMGcB/DUF2drYrtYdCTExM0KTv8/kIDw8HICEhgREjRrBixQri\n4+NJSUkhLi6OlJQUoqKiAP+Rlj//+c+u1N7S4uLiqKurCyx7vV4iIvy7R69evcjNzeXOO+8kOTmZ\n/v37ExERwZ49eygvL+fZZ5+lpqaGmTNn8uSTT7q1CS0qNjaWr77628Hf5mMjKSmJnJwcHnnkERIS\nEkhLSyM8PJx9+/bx+eefs27dOjweD0uXLuW+++5zaxNalOaOYLGxsdTX1weWm+8vKSkpjBs3jtmz\nZ5OYmEjfvn0JDw+nqKiIiooK8vLyOHXqFHPnzuXRRx91axNalPaXYG1lfITqGqQpgM8Y82/AAOA3\nxpibrLVHQ7S+b6W8vBxjDB9//DGJiYmBiyfB/w8aGxtLXl4e0dHR3HbbbRw/fpxx48axb98+Pv74\nY3r27MmRI63mgNj/2iWXXEJxcTGZmZkcPHgw6LRBbW0tHo+H2bNn89VXX/HMM8/Qo0cPVq9ezYAB\nA8jMzMRaS1JSkotb0HIyMzPZtm0bOTk5FBYWkpaWFmirqqqiurqadevWUVtbyx133EFmZiabN28O\nPGfo0KFtJhyB/5qbgoIChg0bhrU26IL1mpoaamtrWbhwIXV1dTz88MP06dOH5557LvCcyZMnt5nJ\nHjR3OGVkZJCfn891101pm7cAABzuSURBVF1HSUkJqampgbbq6mpOnjzJypUr8Xg8zJw5k4yMDNat\nWxd4zujRo11/82tJ2l+CtZXxEZKAZK29+vyfjTH/BUx3OxyB/1NcamoqU6dOJSwsjA0bNjBkyBCq\nqqqw1nLRRRcxbdo0zp07x5YtW/D5fGzdupWbb76ZQYMGcebMGV5//XW3N6PF9O/fn9LSUpYuXQpA\nbm4ub731FvHx8fTr148TJ06wePFiIiIiGDt2LB06dGDMmDGsWbOGd955h6ioKCZNmuTyVrSMkSNH\nkp+fz4QJE/D5fCxcuJC8vDySk5PJzs7m8OHD3HLLLURFRTFnzpzAp8O2avDgwRQVFTFnzhwA7r33\nXl5//XW6d+/OoEGDOHr0KPfddx8RERFMnjy5zfeH5o5gI0aMoKCggGnTpuHz+XjwwQdZu3YtiYmJ\nDBs2jCNHjjBlyhQiIyOZMWNGmx8f2l+CtZXxEebz+UK6gmYB6e9+i23+/PmhLeQC0q1bN7dLaFVG\njhzpdgmtyr59+9wuoVX58MMP3S6hVbnhhhvcLqFVaX60Twj6KReBIUOGhH1TW8h/B8lae02o1yEi\nIiLSkvRL2iIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgi\nIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIi\nIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIi\nIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4Rbhdw3oIFC9wuodV4\n7rnn3C6hVTlx4oTbJbQqH374odsltCrdunVzu4RWpX///m6X0KpUVla6XUKrovn029MRJBEREREH\nBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcF\nJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUk\nEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQR\nERERBwUkEREREQcFJBERERGHCLcL+C55vV5+/OMfU1RURHR0NC+99BK9evUKtC9evJi1a9fSuXNn\nfv7znzN69GiOHj3KpEmTOH36NN27d+fll18mNjbWxa1oOV6vl9///vdUVlYSERHBxIkT6dq1a6B9\ny5Yt7N69m5iYGEaOHEm/fv2oq6tjwYIF9OjRA4D+/ftz7bXXurUJLcbr9bJ06VI++eQToqKieOCB\nB0hMTAy0v/LKK7z55pvExsaSm5vLVVddFWjbu3cvCxYsYMOGDW6UHhJhYWGMGjWKhIQEzp49yxtv\nvEFVVVWgfdiwYaSnp9PY2Eh+fj5//etf6dixI/feey/Hjx8HoLS0lJ07d7q1CS1K+0owr9fLzJkz\nKS4uJjo6mhUrVpCamhpof+KJJ/jDH/5A586dmTVrFjfeeCNHjx5lypQpnDlzhoSEBF544YU2NZc+\n/PDDlJaWEhUVxWOPPUZycnKg/cUXX+RPf/oTcXFx3HXXXUHjoKCggNmzZ7Njxw43Sg+JtjKfhiwg\nGWPCgVWAAc4Bd1hry0K1vm9jw4YNNDQ08P7777Nz505mz57N66+/DkBxcTG/+93v+OCDDwAYOnQo\n2dnZLFq0iNtvv53bbruNhx9+mBdeeIFZs2a5uRkt5qOPPuLMmTPcd999HDx4kNdee43p06cDUFlZ\nye7du7n//vsB/4RnjOHw4cNcccUVjB8/3s3SW9yOHTs4ffo0q1atoqSkhKeffpolS5YAUFZWxtat\nW1m1ahUAd999N5mZmcTExHDs2DHWrl3L2bNn3Sy/xV166aVERETw0ksvkZiYyPXXX8/atWsB6Nq1\nK/369Qv0x9SpUzl48CDdu3enpKSEv/zlL26WHhLaV4Jt3LiRhoYGtm/fzq5du3jggQdYv349ACUl\nJaxfv563334bgOzsbEaMGMETTzzBpEmTmDRpEo899hi/+tWvuPfee93cjBazdetWGhsbWb9+PYWF\nhSxatIjnn38eAGstGzdu5NVXXwVgwoQJDB48mI4dO/L555+Tl5fX5uaPtjKfhvIU2w8BrLVXAfOA\nZSFc17fy7rvvcsMNNwAwePBgdu/eHWjbv38/11xzDTExMcTExNC7d28++ugjli9fTm5uLl6vl8OH\nD9OtWze3ym9xZWVl9O3bF4CePXtSXl4eaDt69Ci9e/cmMjKSyMhI4uPjqayspLy8nMOHD7N8+XJe\neuklampq3Cq/RRUVFZGVlQVAeno6paWlgbZDhw4xcOBAoqOjiY6OJikpiQMHDtDY2MiSJUsCb4xt\nyQ9+8AMOHDgAQEVFReAoCEB8fDyHDh3i7NmznD17lqqqKrp160aPHj3o3r07d9xxB+PHjycuLs6t\n8luc9pVg7733HiNHjgTgyiuv5MMPPwy0WWsZPnx4YC7t1asXJSUlLFmyhFtvvRWv10tFRUXQEbgL\n3Z49exg+fDgAAwYMoLi4ONBWVlZGVlZWYP5ITk7GWktjYyPz5s1j/vz5bpUdMm1lPg1ZQLLWbgCm\nNS0mA8dCta5v69SpU3Tp0iWwHB4eHkiq/fr1Y8eOHdTW1vLll1/y3nvvUVdXR1hYGOfOnSM9PZ3t\n27cHHQq80DU0NNCxY8fAcocOHTh37hwA3//+9zlw4AANDQ14PB4+/fRTGhsb6datG6NGjWLWrFlk\nZGQEPjVe6Orr64Pe0JuPjdTUVAoLC6mrq6Ompobi4mIaGhpYtmwZEydOJD4+3q2yQyY6OpqGhobA\nstfrpUMH/3Rx7NgxkpOTiYqKomPHjiQlJREZGckXX3zB9u3bWb16Nfv37ycnJ8et8luc9pVgtbW1\ndO7cObDcfH+57LLLyM/PD8ylO3fuDJpLBw0axI4dOxgyZIhb5bc4j8dDp06dAsvN+yMtLY2CggI8\nHg/V1dXs3buX+vp6FixYwNSpU0lISHCr7JBpK/NpSK9BstaeNcb8GhgLjAvlur6Nzp07U1tbG1j2\ner1ERPi7oE+fPvzkJz/hxhtvpFevXmRlZfG9730PgMjISPbt28ebb77JbbfdFjh0fKGLiYkJehP0\n+XyEh4cDkJCQwIgRI1ixYgXx8fGkpKQQFxdHSkoKUVFRgP+T0p///GdXam9psbGx1NfXB5abj42U\nlBTGjRvH7NmzSUxMpG/fvoSHh1NUVERFRQV5eXmcOnWKuXPn8uijj7q1CS2qsbGR6OjowHJYWBhe\nrxeAEydOsGvXLnJzc6mqqqKiooL6+noqKys5c+YM4L/+KDs725XaQ0H7SrBOnTrh8XgCy833l0sv\nvZTp06czduxYLrnkEgYNGsTFF18M+OfSPXv2sG3bNu666y42b97sSv0tLS4ujrq6usBy8/7o1asX\nubm53HnnnSQnJ9O/f38iIiLYs2cP5eXlPPvss9TU1DBz5kyefPJJtzahRbWV+TTk32Kz1t4OpAGr\njDH/Eur1/T1XXXVV4PqInTt30q9fv0DbF198wYkTJ3j33Xd56qmnOHz4MOnp6fz4xz9m+/btgH9S\nOP8pui245JJL+PjjjwE4ePBg0GmU2tpaPB4Ps2fP5kc/+hHV1dX06NGD3/72t+zduxfwH0pPSkpy\npfaWlpGRwfvvvw/4r6FofsFpdXU1J0+eZOXKlcycOZPjx4+TkZHBunXrWLFiBStWrKBz586u78wt\nqby8nN69ewOQmJgYuPAa/JNfbGwseXl5bNq0iS5dunD8+HHGjBkTdBrqyJEjrtQeCtpXgg0ZMiQQ\nbnbt2sVll10WaDs/l7755ps8/vjjVFRUcNlllzFz5szAh8u2NpdmZmYGtq2wsJC0tLRAW1VVFdXV\n1axbt46HHnqIzz//nMzMTDZv3syaNWtYs2YNXbp0aTPhCNrOfBrKi7T/A0i01i4E6gEv/ou1XTN2\n7Fi2bt3K0KFD8fl8rF69mmXLltGrVy9++MMf8umnnzJo0CCioqJ4/PHHCQ8P56c//SnTp0/nkUce\noUOHDjz33HNubkKL6t+/P6WlpSxduhSA3Nxc3nrrLeLj4+nXrx8nTpxg8eLFREREMHbsWDp06MCY\nMWNYs2YN77zzDlFRUUyaNMnlrWgZI0aMoKCggGnTpuHz+XjwwQdZu3YtiYmJDBs2jCNHjjBlyhQi\nIyOZMWNG4OhBW1VaWkpqaipTp04lLCyMDRs2MGTIEKqqqrDWctFFFzFt2jTOnTvHli1b8Pl8bN26\nlZtvvplBgwZx5syZwBcg2gLtK8Fuuukmtm3bRnZ2Nj6fj5UrV/L000+TmppKTk4Ohw4dYvjw4YFv\ndIWHh3PPPffws5/9jEWLFhEWFsby5cvd3owWM3LkSPLz85kwYQI+n4+FCxeSl5dHcnIy2dnZHD58\nmFtuuYWoqCjmzJnT5uePtjKfhvl8vpC8cNPRotVAAhAJLLLW/r0ZMzSFXIDaUghrCQMHDnS7hFbl\nP//zP90uoVVpS1+caAmTJ092u4RWpbKy0u0SWpUTJ064XUKrMmTIkLBvagvZESRrbR3Q9r7fKiIi\nIm1e2zkJLCIiItJCFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBE\nREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkERE\nREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERE\nRBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBzCfD6f2zUAcPz48dZR\nSCtQU1PjdgmtSpcuXdwuQUQuUGVlZW6X0Kqkpqa6XUKr0rVr17BvatMRJBEREREHBSQRERERBwUk\nEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQR\nERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBER\nEREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkEREREQcFJBEREREHBSQRERERBwUkERER\nEQcFJBERERGHdhWQvF4v999/Pzk5OYwdO5aDBw8GtT/zzDNkZ2czZswYtmzZEtT2/vvvM3DgwO+y\n3JDyer3MmzeP8ePHk5uby2effRbU/uKLL3LTTTcxceJEtm/fHtRWUFDA1Vdf/V2WG3IaG8HUH8HU\nH8HUH8G8Xi9LlizhrrvuYsaMGVRUVAS1v/LKK9x+++3cc8895OfnB7Xt3buXm2+++bssN+TayviI\nCNULG2MigTwgBYgGfmmtfSNU6/s2Nm3aRGNjI3/5y1/YvXs38+fP5ze/+Q0A+/bt47XXXmPTpk0A\njB49mmHDhhEbG0tlZSXPP/88Z86ccbP8FrV161YaGxtZv349hYWFLFq0iOeffx4Aay0bN27k1Vdf\nBWDChAkMHjyYjh078vnnn5OXl8fZs2fdLL/FaWwEU38EU38EU38E27FjB6dPn2bVqlWUlJTw9NNP\ns2TJEgDKysrYunUrq1atAuDuu+8mMzOTmJgYjh07xtq1azWfttLxEcojSLnAl9ba4cCNwLMhXNe3\n8sEHH3DttdcCcMUVV1BUVBRo++STTxg6dCgxMTHExMTQs2dP9u3bR0NDAz//+c9ZvHixW2WHxJ49\nexg+fDgAAwYMoLi4ONBWVlZGVlYW0dHRREdHk5ycjLWWxsZG5s2bx/z5890qO2Q0NoKpP4KpP4Kp\nP4IVFRWRlZUFQHp6OqWlpYG2Q4cOMXDgwMB8mpSUxIEDB2hsbGTJkiXcf//9bpUdMm1lfIQyIP0B\nmNts2fWIXFtbS+fOnQPL4eHhgeTep08fdu7cicfjoaqqit27d1NfX88DDzzAPffcQ/fu3d0qOyQ8\nHg+dOnUKLDfvi7S0NAoKCvB4PFRXV7N3717q6+tZsGABU6dOJSEhwa2yQ0ZjI5j6I5j6I5j6I1h9\nfT1xcXGB5eb9kZqaSmFhIXV1ddTU1FBcXExDQwPLli1j4sSJxMfHu1V2yLSV8RGyU2zWWg+AMaYT\n8CrwUKjW9W116tQJj8cTWPZ6vURE+LsgLS2NKVOmcOutt9KzZ08uv/xyIiIi+OCDDzh06BBPPPEE\nJ0+eZNq0abz44otubUKLiYuLo66uLrDcvC969epFbm4ud955J8nJyfTv35+IiAj27NlDeXk5zz77\nLDU1NcycOZMnn3zSrU1oURobwdQfwdQfwdQfwWJjY6mvrw8sN++PlJQUxo0bx+zZs0lMTKRv376E\nh4dTVFRERUUFeXl5nDp1irlz5/Loo4+6tQktqq2Mj5AFJABjTBLwR+A5a+3vQrmub+PKK69ky5Yt\njBkzht27d9OnT59A24kTJ6iqqmLjxo2cOnWK8ePHk5WVxXvvvRd4Tnp6uuv/YC0lMzOTbdu2kZOT\nQ2FhIWlpaYG2qqoqqqurWbduHbW1tdxxxx1kZmayefPmwHOGDh3aZsIRaGw4qT+CqT+CqT+CZWRk\nkJ+fz3XXXUdJSQmpqamBturqak6ePMnKlSvxeDzMnDmTjIwM1q1bF3jO6NGj20w4grYzPkJ5kXY3\nYAvwE2vtW6Faz/9ETk4Ob7/9NqNGjcLn8/HUU0+xcuVKUlJSuP766/nss8+4/vrriYyMZP78+YSH\nh7tdcsiMHDmS/Px8JkyYgM/nY+HCheTl5ZGcnEx2djaHDx/mlltuISoqijlz5rTpvgCNDSf1RzD1\nRzD1R7ARI0ZQUFDAtGnT8Pl8PPjgg6xdu5bExESGDRvGkSNHmDJlCpGRkcyYMaPN90dbGR9hPp8v\nJC9sjHkKmACUNnv4RmvtV1/3/OPHj4emkAtQTU2N2yW0Kl26dHG7BBG5QJWVlbldQqvS/OiWQNeu\nXcO+qS2U1yD9DPhZqF5fREREJFTa1Q9FioiIiHwbCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIi\nDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIO\nCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4K\nSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpIIiIiIg4KSCIiIiIOCkgiIiIiDgpI\nIiIiIg5hPp/P7RpEREREWhUdQRIRERFxUEASERERcVBAEhEREXFQQBIRERFxUEASERERcVBAEhER\nEXGIcLsANxljIoE8IAWIBn4J7ANeBnxACTDDWut1qcTvlDEmHFgFGOAccAcQRjvtDwBjTFdgDzAS\nOEs77gsAY8xeoKZp8SDwAvAU/r7ZYq1d4FZt3zVjzAPATUAU8BzwNu14fBhjJgOTmxZjgAHANbTD\n8dH03vJr/O8t54C7aMfzhzEmGlgNXAKcAmYAF9PKx0Z7P4KUC3xprR0O3Ag8CywDHmp6LAwY42J9\n37UfAlhrrwLm4e+LdtsfTZPcC8BXTQ+1274AMMbEAFhrr2n67w5gJTARGAZkGWMud7PG74ox5hpg\nKHAVMAJIop2PD2vty+fHBv4PFT+lnY4PIAeIsNYOBR4BHqN9j4+7AI+1djBwL/732lY/Ntp7QPoD\nMLfZ8lkgE/8nQYBNwL9910W5xVq7AZjWtJgMHKMd9wewFP9OfKRpuT33BUB/INYYs8UYs80YczUQ\nba0ts9b6gM3Ade6W+J25HigG/ghsBP6ExgcAxpgrgMuAdbTf8fFXIMIY0wHoDJyhfY+Pvvi3GWut\nBQZxAYyNdh2QrLUea22tMaYT8CrwEBDW9A8GUAt0ca1AF1hrzxpjfg08g79P2mV/NJ0u+MJau7nZ\nw+2yL5qpxx8arwem4z9kXt+svT31yfeAK4Af4e+L3wId2vn4OO8XwAL8weBUs8fbU5948J9eK8V/\n2cLTtO/5oxAYbYwJM8YMxr/tnmbtrbI/2nVAAjDGJAHbgVestb8Dmp8T7gScdKUwF1lrbwfS8O/Y\nHZs1taf+mAKMNMb8F/5rKX4DdG3W3p764ry/AmustT5r7V/xX4v0r83a21OffAlsttaebvpE3EDw\nBN+e+iLAGPN/gEuttdvxh6NOzZrbU5/Mwj8+0vAfef01/mvVzmtPfQH+a31P4X+v/SFQBPxLs/ZW\n2R/tOiAZY7oBW4A51tq8pof3Nl1fAP7rkt5xozY3GGP+o+nCU/AfGfACu9tjf1hrr7bWjmi6nqIQ\nuA3Y1B77opkpwBMAxpgeQCxQZ4xJNcaE4T+y1F765F3ghqZPxD3wT/ZvtfPxAXA18CaAtfYUcLqd\njo9q/vZlhiogknb83oL/lNq7TfPpH/F/2Gr1Y6Ndf4sN/6Hgi4C5xpjz1yL9DHjaGBMF7Md/mqm9\neA1YbYzZgX+Hnom/D1a10/5wmk377otfAS8bY97F/02cKfhD9G+BcPzfRPnAxfq+M9baPzVdg7UL\n/wfNGfi/1deexwf4vwH7abPl86cf29X4AJYDecaYd/AfOfoFsJv2Oz4+AR41xtyH/0jRVOAHtPKx\nEebz+f7xs0RERETakXZ9ik1ERETk6yggiYiIiDgoIImIiIg4KCCJiIiIOCggiYiIiDgoIImIiIg4\nKCCJSKtnjLnSGLO46c83GWMeacnXFBFxau8/FCkiF4a+QDcAa+0bwBst+ZoiIk76oUgRaRFNt1H4\nBf7b1PTBf7f7idba09/w/BuAR/D/avtB4C5r7ZfGmKXASPy/0r0BeAr4CIjDf6uTSuAaa+1kY8wh\nYG3T888Cj+L/xfPewGxr7XpjTDr+my/H4b+f3kL8d5pv/poLgSfx31Hch//ejIubtmkJ/l/7LcF/\nT74lTc+pBm611p743/WciLRGOsUmIi1pKPAT/AHpB/jvsfTfGGPigUXA9dbagcBmYLExJhm40Vrb\nH7gK/1GeBmAe8Ia19rGvebmj1tor8N++4f8B/xfIBc7fV/BO4JfW2kHAtcDj1tqTjtecDiQBGcCV\nwL8bY0Y1/f00ILvpJs4PAdOb1rcVuPyf6CMRuQAoIIlISyqx1lZYa734A8u/fsPzsvAHqO3GmEL8\noao3/qNDXxlj8vHfF3GOtbbhH6xzU9P/PwPettaebfrzRU2PzwZimm7E/Ev8R42csoGXrbXnrLX1\n+O8RdV1Tm7XWnr/x6BvAH40xzwJ7rbVb/kFtInKBUkASkZbUPMz4gLBveF44/rt7D7DWDsB/t+9/\nbwo3WcBc4GLgfWNM2j9YZ/NTeGe/pn09MBbYBzz4Da/hnAvD+Ns1ml+df9Bauxy4BjgALDHGfNPr\nicgFTgFJRNzwATCkWfiZCyw1xgwE3gZ2WGvvwx9qDP7g889+qWQkMM9a+zpwI4AxJtzxmtuA240x\n4caYWGASsN35QsaYD4BO1ton8d+xXafYRNooBSQR+c5Za48CU4D1xphi/EFjtrV2L/A+UGKM+RB/\nQNoE7AIGG2MW/ROrexh41xizDxgOHAJ6Ol7zBaACKAL2AhuttX/8mtf6BfCyMWYPMBX/NU8i0gbp\nW2wiIiIiDvodJBEJCWNMR/xHg77OvKbfMxIRaZV0BElERETEQdcgiYiIiDgoIImIiIg4KCCJiIiI\nOCggiYiIiDgoIImIiIg4/H90pkK95bJ+SAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x720 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#各参数组合的scores\n",
    "scores = np.array(result.mean_test_score).reshape(8,8)\n",
    "plt.figure(figsize=(12,10))\n",
    "mglearn.tools.heatmap(scores,xlabel=\"n_estimators\",ylabel=\"max_depth\",xticklabels=param_grid[\"n_estimators\"],\n",
    "                      yticklabels=param_grid[\"max_depth\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估指标与评分----混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 60   3]\n",
      " [  4 104]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcMAAAD8CAYAAAAL3c8SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzt3Xl8FPX9x/EXuZCrAhIrSCGo8A0I\nyqECigK1Jx4VqXKJhBu1UkRFCVZBEBQoImA4RH+AlSqKgFqtVqQUihcgWqr9qlSkqGhaBYFEAkl+\nf8wsLrl2NuxmdrPv5+ORB5vMZ2c+u8zuZ7+f+c5sjeLiYkRERBJZkt8JiIiI+E3FUEREEp6KoYiI\nJDwVQxERSXgqhiIikvBS/E5Awpebe0BTgCOoQYPafPNNnt9pVAudOrUFYOvWHT5nUj1o34ys9PR6\nNcpbppGhJLyUlGS/UxApk/bNqqNiKCIiCU/FUEREEp6OGYpIxPTs+RO/UxCplBq6HFv80QSayEpP\nr0du7gG/0xApRftmZGkCjYiISAVUDEUkYubNm8O8eXP8TkMkbCqGIhIxS5cuYenSJX6nIRI2FUMR\nEUl4KoYiIpLwVAxFRCThqRiKiEjC00n3IhIxtWvX9jsFkUpRMRSRiNm48S2/UxCpFLVJRUQk4Wlk\nKAJs27aFMWNGh4zr0KETc+cupEaNcq/q5Isrr/w5X3/9vwpjnn76ORo3bhLVPLZscUaG5513QVS3\nU57f/GYk27dv82XbkXDNNf357W9v9TuNhKRiKNx33yReeukFv9OIqkgVgnfe2cqzzz5Nnz7XRiCr\n6mfUqKGAvty3Knzxxedcc82VVba9pKQk9yeZ1NQUTjqpFrVq1aJ27To0aNCQBg0a0LhxE5o2/REt\nWxoyMlqQlBQ/zUcVQ5EwLVw4j65dL6JJk9P9TkWkyhQVFVFUVAQcpaDgMIcOHaowvnbtOrRtew6X\nXNKdbt160KhRo6pJtJLip2yLxIj8/HymT78XfeOLSPny8g7x1luvM2vW/fTpcxm/+92d7NjxD7/T\nKpeKoUglBNqlIhJaYWEh69e/yujRQ7j77gl8+eVev1MqRcVQpJIWLpzH559/5ncaInHltdf+wuDB\n/di06W9+p3IcFUORSlK7VKRyDh48yIQJt7Jy5Qq/UzlGE2hEToBmlx7vscce9zsFiRPFxcXMnTub\n1NQ0evf+td/pqBiKnCjNLv3eued28DuFkNq378j8+Yv9TkNcDz44g+bNM+jY8Txf81AxFCZOnMTE\niZOitv5u3ULv5EOGjGDYsFFRyyGa8vPzuf/+KTz00IKYOxlfJDv7Hnr1uiIi6yosLDz28913+eTl\n5XHgwLfk5n7Fnj17eP/9HWzevJHDhw97XmdRURFTptzNsmV/5Ac/ODkieVaGiqFIBGzbtoXVq5/h\n6quv8TsVX3Xu3B6AN9/c7nMmEg3JyckkJycDUKtWLRo0aAhAZmabYzGHDh3kuefWsGzZEg4ePOhp\nvbm5X7FkyULGjbsj8kl7pAk0IhGyYMHchJ9devToUY4ePep3GuKjOnXq0r//daxYsYp27c71fL+1\na59l165PophZxVQMRSIk0C7V7FIRaNjwFB56aAFdu17kKb6wsJAVK5ZHOavyqRiKRFCgXSoikJaW\nxuTJ0zjjjDM9xb/66ivs378vylmVTcVQJMLULhX5Xu3adcjOnuTpot0FBYdZv35dFWRVmoqhiEcN\nG55CSkroOWdql4ocLzOzNb/4xWWeYt98c3OUsymbiqGIR6ed1phBg4Z4ik3UdunIkTcwcuQNfqch\nMcjrTOstW952vx2jaqkYioRh8OBhnHVWK0+xidguHTXqJkaNusnvNCQGZWa2oXnzjJBx+fl5fPHF\n59FPqAQVQ5EwpKSkMHHiPcfOtaqI2qUix2vb9hxPcX6cYqFiKBKmli0N118/1FNsorVLx469ibFj\nNTKUsrVqlekp7n//+2+UMylNxVCkEgYPHsaZZ7b0FLtgwTxf2j5+2LhxAxs3bvA7DYlRgSvWhJKX\ndyjKmZSmYihSCeG1S/P0VU8iQL16dT3Ffffdd1HOpDQVQ5FKatUqU7NLRcKQn++tyKWl1YxyJqWp\nGIqcgKys4WqXini0b983nuJq164V5UxKUzEUOQFql4p49803X3uKa9To1ChnUpqKocgJCrddumbN\nqihn5J927c4N65sKJLHs2PGep7gmTZpEOZPSVAxFIsBpl57lKTYnZ261bZcuXfoES5c+4XcaEoOO\nHDnC9u3bQsbVrFmTZs0yop9QCSqGIhGQkpJCdvYktUtFyvHGG38nPz8/ZFzr1md7ugZwpKkYikSI\nMZlcd12Wp9jq2i598sknePJJjQzleEVFRSxZsshTbJcuF0Y5m7KpGIpEUKK3S2fOnM7MmdP9TkNi\nzIsvPs/OnR95iu3R49IoZ1M2FUORCEpNTQ2rXaprl0p1995725k9e4an2E6dzqdp0x9FOaOyVX1j\nVqSaMyaTgQMHs3z5YyFjt259mzVrVtG796+rIDMB2L59G926ned3GkybNotLLunhdxpRZe2/uPPO\nWykoOOwpvm/fgVHOqHwaGYpEwZAhIxK6XSqJ7ejRo6xY8Tg33DCMb7/d7+k+55zTngsv7BblzMqn\nYigSBampqUyY4P1kfLVLpTr48su9rFixnH79epOT85DnEWFycjJjxtwa5ewqFhdtUmNMbSDLWpvj\n/j4JuM5a6+2jd2RzWQo0tdb+pKq3LfElM7O12qVSrRQWFnL06BHy8vLZv38fublfsWfPf/j44w/Z\nseM9du78uFLrHTx4GJmZrSOcbXjiohgCtwAjgBz391nAfP/SEfFmyJARbNq0gX//e2fI2JycuXTp\nciGNG1f91TciZcOGN/xOQUqYNm0y06ZN9juNcp1/fmcGDx7mdxpx0yatEfyLtfagtbbqv/1RJEyJ\nNru0bt261K3r7Wt6RFq0OIMpUx7w9PqItpAjQ2NMMTAMyALOBz4FZltrFwfFjABuA5oBHwOzrLXL\ngpZ3Bh4EOgC7cEZ2S4AW1tpdxpgMYCbQEzgZ+Ax42Fo70xiTBUwJyqUn0AO3TWqM2QB8Yq3NCtre\ntcBS4DRr7beh8ivjMf8QmA38EqcQvwqMsdZ+UUZsH+BOoC1QDLwDjLXWvu0uzwLuAM4A9rp5TbbW\nFrnbWQB0B04CXgdus9ZuLy83iT+Zma0ZMOB6Hn/8/0LGxnu7dNeuTwDIyGjhcyYS61q3bsPMmXNj\n5sOT15HhAzhtyQ7ARmCBMaY5gDHmBuA+YCJOQXgAeMgYM9hdfjrwF+CfQEfgd25MsOdxikFPoDWw\nHJhhjGkPPOXG7wEaA5tL3HcZ0NsYc1LQ3wYAq91CWGF+JRljUoBXcIpXL+AS4DSg1JfRGWPOB1bi\nFLjWOEWtBvCIu/wcYJG77ZbAWOB24Dp3FTlAKtDNfW4OANXvsiTC0KEjadHiDE+x8Ty7tE+fK+jT\n5wq/05AYd9llVzJ37iLq16/vdyrHeD1m+Ji1diWAMeZ2YDhwAc4ocSLOSCdQLHa6hTIbp1CNBP4L\njLbWFgIfGGNOA+a566uFU0yetNZ+5v5tsrvedtba7caYg0ChtXavuzw4t6fddfUCnjXGNMAZ0V3p\nLg+VX0mXAucAZ1hrP3G3NxwYUqLgAhwBbrTWBq4ztMsY8wjOqBfgTJzR4qfW2t3AbmPMT3AKO8BZ\nwHs4I9vvjDGjgDbGmCRrbVEZuQHQoEFtUlL8bytEUp06NUlPr+fb9uvXrx0yJjU1+YRynDlzBv36\n9ePo0aMVxuXn5/H7309j6dKl1KhRo8LYgKSk0HENG9aJ+nMcyMOv/8u0tNBvaRdccAGPP/54FWQT\nGeU9l4cP16niTE5cq1atuP3227nkkkv8TqUUr8Xww8ANa+0+txilGWPSgdOBWcaY4NFeCpBijEnD\nGfG87RbCgE1B68s3xswH+hpjLsAZQbXHGbWGfMe31h4wxqwG+gPPAtfgFN9XveRnrS0oscp2QG6g\nELrbsDit0OMKsVuo9xljJgBtSuQO8GfgTWCLMeZj4GXgKbcwgtP+XQ70cdu9LwHLKyqEAN98kxfq\naYk7hw4dJjf3gC/bTk+vx759oZ/TI0cKTyjH007LoH//QZ7apW+88QZLlizlqqu8tUuLikIfZ/z6\n60PUrBnd5ziQh1//lwUFFX/QCMT4lV+40tPrlZvr118fquJsKicpKYn27TvSp8+1XHJJT2rUqOHr\na708XtukZZ0sUgMIFJKbcYpA4KctkAkcdX/K3Y4xpg7wBk77MBdYDHQCKiwIJSwDLjfG1MNpkT7h\nFl8v+ZV0xOtGjTE9gX+563wbmAD8NrDcWptvre2Oc6x1OU6beYMxZry7/BmgCc5Iey9OC/kd91ii\nVEPhtEsffjh+26WS2OrWrcdFF13MmDHjWLXqBebOXUj37j/23OnwwwmdWmGt3W+M+QzIsNYeO8HE\nGDMaaG+tHW2MeQ8YZIxJDhoddg5aTQ+cYnKKtfZr9/4Gp4AGnrlQH3vX4YwGhwIXA7/xml8Z6/oA\nSDfGNLfWfurGZ+IcKz2/ROyNwCvW2r5B6/6Z+28N4CdAF2vtFGALMMUYkwNkGWNmA/cDf7DWrgBW\nGGNOBb7EOfa4MsRjljiUmprKxImTGDVqCIWFhRXGBmaXzpmTE9NvIpI4kpOTSUtLIzU1jZNOOon6\n9etTv35DTjnlFE4/vSkZGS3IyDiDZs2ak5QULycrOCJxnuFUYLYxZjdOUeqMMxMzcGXWHOBWIMcY\n8yDQCnd2KE6Ry3VvDzTGPIdzHG22+7ea7r8HgAZukfy0ZALuzMw/uOvdbq3dEUZ+Jb0KbAOWG2PG\n4Ywe5wMfuDNfg2NzgcuMMV1witjlOJNkArkXAPcYY/bjTBI6DWeS0BvW2qPGmI7AxcaYMcBXwECc\nkWnob8CUuJWZ2Sas2aVr167y3C4VKSk7+x569dKkplBOuHRbaxfitAdvB97HKT5TgMnu8i9xJrec\nD7wLTOf7k+cLrLVvufe9A6fl+DDwB2A934/EVuGckvEecFk5qSwD6uG0Iz3nV8bjKQJ+hTPS/Cvw\nGrAbKOvd6G6cUyleBrYCVwOBWarnW2s34IxWR7rbXgNsAMa4MQOAT4AXcEakVwG/Ch7FSvU0ZMiI\nsNqle/eWOqsnJs2YMZsZM2aHDhSJMSFHhtbaUv2Zkn+z1j4EPFTW/Y0xbYAj1tqOQX/rh3McMte9\n/yyccw+DzQxa/2fAuUHLVgGTSuTwL0qcnO8lv3Li9wB9ylmWFXT7vzgFrKSVQTHLKVGgg5btBfp5\nzUuqj7S0NLKz72H06KGe2qXTp09hzpyHY75deumlP/M7BZFKqYqm7o+A9caYvsaYZsaY7jijsqes\ntaGnfolUU61bn03//oM8xW7d+hZr1+oUVJFoiXoxtNa+jHP1l3txTtFYAbwI3BDtbYvEuqFDR5KR\nUX3apZdf/jMuv1yjQ4k/VXKhbmvtfHRhbZFS0tLSmDix+rRLdSqIxKv4mvsqUg21bn02/fpdFzoQ\ntUtFokXFUCQGDBs2ynO7NCdnXsy3S0XijYqhSAxwZpfe7emrbPLyDjF9+pSQcSLinYqhSIxo06Zt\nWO3SNWvULhWJFBVDkRjitEu9fRdgTk7szS7t3fvXcftdjJLYVAxFYkjgZPx4bZfeddck7rprkt9p\niIRNxVAkxrRp05a+fQd6ilW7VCQyVAxFYtDw4aPDapfm58fGd1xOnTqJqVMn+ZqDSGWoGIrEoLS0\nNCZM8N4uzc/Pr4KsQlu9+hlWr37G7zREwqZiKBKjzj7be7tURE6MiqFIDAunXSoiladiKBLDwmmX\nikjlqRiKxDinXTrA7zREqjUVQ5E4MGzYaJo3z/A7jZAaN25C48ZN/E5DJGwqhiJxoGbNmkyYcA9J\nSbH9kn3hhVd44YVX/E5DJGyx/coSkWPatm2n2aUiUaJiKBJHhg+P7XbpunWvsG6dRoYSf1QMReJI\nrLdLx48fx/jx4/xOQyRssfmKEpFyqV0qEnkqhiJxaPjw0TRr1tzvNESqDRVDkThUs2ZNsrNjt10q\nEm/0ShKJU23bnsO11+pkfJFISPE7ARGpvBEjbmDz5o3s3v2p36nEhF69rqBDh04VxuiiAFKWGsXF\nxX7nIGHKzT2g/7QISk+vR27uAb/TqBZ27foEQBcXjxDtm5GVnl6vRnnLNDIUkYhREZR4pWOGIhIx\nBw8e5ODBg36nIRI2jQxFJGK6d+8CwNatO3zORCQ8GhmKiEjCUzEUEZGEp2IoIiIJT8VQREQSnoqh\niIgkPM0mFZGIuf32CX6nIFIpKoYiEjH9+umrpSQ+qU0qIiIJT8VQRCImK2sgWVkaHUr8UZtURCLm\nH/941+8URCpFI0MREUl4KoYiIpLwVAxFRCThqRiKiEjC0wQaEYmYiy/u7ncKIpWiYigiETNnzsN+\npyBSKTWKi4v9zkHC1KxZ8zL/0268cQzDho10b4/gzTdfLxXTqdN5LF68FIDHH1/KnDmzytzG669v\nIy0tjY8++pB+/a4uM2b27Hl0794TgJ//vAf//e9/S8Vce21/7rhjIgD33DORF15YW9bjYfXqPwHw\n0kt/4q677ihze88//zJNmpzOvn3fcOmlF5cZk519N336XAvAwIHX8K9/fVAqpmfPnzBr1hwA5s2b\nw/Llj1JUdPxTWrt2bTZufAuALVveYtSooWVu77HHHufcczsA0Llze44ePVoqZuTIGxg16iYAxo69\niY0bN5SKadfuXJYufQKAJ598gpkzp5e5vQ0b3qBu3brs2vUJffpcUWbMjBmzufTSnwFw+eU/44sv\nPi8V07v3r7nrrkkATJ06idWrnykV07hxE1544RUA1q17hfHjx5W5vVWrnicjowUHDx489uW+Jd1+\n+4RjV6fJyhpY5ikYF1/c/VgxXbToYRYvXlAqJiUlhTff3A7Au+++w9Chg8rc3qJFj3HeeRe4672A\nvLy8UjFZWcO5+eaxANx221jWr3+1VExmZmueeOJp93GuZNq0e8vc3rp1G6lfvwGff/4ZV1zx8zJj\npk59gF/+8jIAeve+jN27Py0Vc/nlv2Ly5PsAeOCB+3jmmSdL7ZuNGjXi5Zf/CsCGDesZN+7mMrf3\n5JPP0rJlKwoKCujatWOZMWPH3sagQVkAjByZxdatW0rFdO7clZycRwB49NHF5OTMLXNdgS90fv/9\nfzJoUN8yY+bPX0TXrhcB0LPnRXz77f5SMQMHXs+4ceMByM6+nZdffqlUzJlnnsXKlWsAeP75NUya\ndFeZ23vppdc49dRT+eqrr/jlL3/M7t2f1igzEB0zFJEIOnDgWw4c+NbvNETCppFhHMrNPaD/tAhK\nT69Hbu4Bv9OoFjp1agt8P0qQE6N9M7LS0+tpZCgiIlIeFUMREUl4KoYiIpLwVAxFRCTh6TxDEYmY\nlBS9pUh80p4rIhETOAdQJN6oTSoiIglPxVBEIubdd9/h3Xff8TsNkbCpTSoiERO4PJpOupd4o5Gh\niIgkPBVDERFJeCqGIiKS8FQMRUQk4akYiohIwtNsUinXb34zku3bt/mdRqVdc01/fvvbWz3Fbtu2\nhTFjRoeM69ChE3PnLqRGjXK/CcYXV175c77++n8Vxjz99HM0btwkqnksWvRYVNcvEi0qhiJheOed\nrTz77NP06XOt36nEpMC3y0fbffdN4qWXXqiSbfnl6aefIz3dlLv8iy8+55prrqyyfJKSktyfZFJT\nUzjppFrUqlWL2rXr0KBBQxo0aEDjxk1o2vRHtGxpyMhoQVJS/DQfVQxFwrRw4Ty6dr2IJk1O9zsV\nkSpTVFREUVERcJSCgsMcOnSowvjatevQtu05XHJJd7p160GjRo2qJtFKip+yLRIj8vPzmT79XoqL\ni/1OJeZcfPEFXHxx1YwOJbbl5R3irbdeZ9as++nT5zJ+97s72bHjH36nVS4VQ5FKCLRL5Xh5eXnk\n5eX5nYbEmMLCQtavf5XRo4dw990T+PLLvX6nVIqKoUglLVw4j88//8zvNETiymuv/YXBg/uxadPf\n/E7lOCqGIpWkdqlI5Rw8eJAJE25l5coVfqdyjIqhyAlQu1SkcoqLi5k7dzarVz/jdyqAZpPKCWrf\nviPz5y/2Ow1faXapSOU9+OAMmjfPoGPH83zNQ8VQ5ATl5+dz//1TeOihBTF3Mn5Vy8oaXiXbmThx\nEhMnTora+rt1C/3GPGTICIYNGxW1HCIlO/seevW6IiLrKiwsPPbz3Xf55OXlceDAt+TmfsWePXt4\n//0dbN68kcOHD3teZ1FREVOm3M2yZX/kBz84OSJ5VoaKoUgEbNu2hdWrn+Hqq6/xOxVf3XzzWL9T\nkChKTk4mOTkZgFq1atGgQUMAMjPbHIs5dOggzz23hmXLlnDw4EFP683N/YolSxYybtwdkU/aIx0z\nFImQBQvmanapJLw6derSv/91rFixinbtzvV8v7Vrn2XXrk+imFnFVAxFIiTQLk3k2aW33TaW227T\n6FCgYcNTeOihBXTtepGn+MLCQlasWB7lrMqnYigSQYF2aaJav/5V1q9/1e80JEakpaUxefI0zjjj\nTE/xr776Cvv374tyVmVTMRSJMLVLRb5Xu3YdsrMnebpod0HBYdavX1cFWZWmYijiUcOGp5CSEnrO\nmdqlIsfLzGzNL35xmafYN9/cHOVsyqZiKOLRaac1ZtCgIZ5iE71dKlKS15nWW7a87X47RtVSMRQJ\nw+DBwzjrrFaeYtUuFfleZmYbmjfPCBmXn5/HF198Hv2ESlAxFAlDSkoKEyfec+xcq4okYrs0M7M1\nmZmt/U5DYlTbtud4ivPjFAsVQ5EwtWxpuP76oZ5iE61d+sQTT/PEE7pWq5StVatMT3H/+99/o5xJ\naSqGIpUwePAwzjyzpafYBQvm+dL2EYk1gSvWhJKXdyjKmZSmYihSCeG1S/MS5queVq1ayapVK/1O\nQ2JUvXp1PcV99913Uc6kNBVDkUpq1SpTs0tLmDbtXqZNu9fvNCRG5ed7K3JpaTWjnElpKoYiJyAr\na7japSIe7dv3jae42rVrRTmT0lQMRU6A2qUi3n3zzdee4ho1OjXKmZSmYihygsJtl65ZsyrKGYnE\nph073vMU16RJkyhnUpqKoUgEOO3SszzF5uTMVbtUEs6RI0fYvn1byLiaNWvSrFlG9BMqQcVQJAJS\nUlLIzp6kdqlIOd544+/k5+eHjGvd+mxP1wCONBVDkQgxJpPrrsvyFFtd26Xr1m1k3bqNfqchMaao\nqIglSxZ5iu3S5cIoZ1M2FUORCEr0dmn9+g2oX7+B32lIjHnxxefZufMjT7E9elwa5WzKpmIoEkGp\nqalhtUur27VLP//8M12cXI7z3nvbmT17hqfYTp3Op2nTH0U5o7JVfWNWqpXt27fRrdt5fqfBtGmz\nuOSSHn6nATjt0oEDB7N8+WMhY7dufZs1a1bRu/evqyCz6Lviip8DsHXrDp8zkVhg7b+4885bKSg4\n7Cm+b9+BUc6ofBoZikTBkCEjErpdKont6NGjrFjxODfcMIxvv93v6T7nnNOeCy/sFuXMyqdiKBIF\nqampTJjg/WT86tYulcT05Zd7WbFiOf369SYn5yHPI8Lk5GTGjLk1ytlVLGbbpMaYXcASa+1UY0wK\n8Btr7Rx3WZa7rMrzN8ZMAq6z1nr72C8JKzOzdcK2S6V6Kiws5OjRI+Tl5bN//z5yc79iz57/8PHH\nH7Jjx3vs3PlxpdY7ePAw378HM2aLIXA+kOfe7gs8CMxxf38KeMmPpETCMWTICDZt2sC//70zZGxO\nzly6dLmQxo2r/uobUn1NmzaZadMm+51Guc4/vzODBw/zO43YbZNaa3OttYEvtapRYlm+tfZLH9IS\nCUuizy4VqUiLFmcwZcoDnl4f0VbpkaExphgYDYwAzgb+CYyz1v4tKGYoMA44E/gMmGOtne8uqwPM\nB3oBJwPbgWxr7Wvu8l3AEmAT8HjQNgMXgVxirU0xxiwDmltrewRt9wLgTaCltfZjY8xVwGTAAIH1\nzrbWFpXz2OoB9wO/BmoDm4Ex1lpbRmwPd92dgFTgA+BOa+2f3eWXAVOBTGAf8Axwu7X2u1DPgVQP\nmZmtGTDgeh5//P9CxsZ7u3Tq1Af8TkHiROvWbZg5cy5163r7jsNoO9E26UzgVmAjcAvwsjHmbGvt\nv40x43CKwBhgA/BjYI4xpqa19vfAvUAb4OfAt8DtwBpjTOOgESE4heg3OEWjMbAfp20asAz4izHm\ndGtt4ASnAcBmtxD2Ap4AbnbzOBt4GKiDU8TKshJoAfTHKeLT3Md23Hf1GGN+hNOu/T1Oka7rPubl\nxpimwA+AZ93n4M84HwqeBHLdx+/1OThOgwa1SUmJ/ieptLRY7qIf7+STa5GeXq/S969fv3bImNTU\n5EpvY/z4cbzxxiY++ij0iccLFsylV6+f0rRpU8/rT0qqETKmYcM6J/QceXH99f2iuv5YUqdOzag/\nn0C52zh8uE7Utx0tffr04a677qJ27dCvu6pyou92j1hrHwEwxtwI/BQYYYzJBsbjjASXuLEfGWPO\nAO4wxswGzgIOAJ9Ya/cbY24DVgGFwRuw1hYYY/a7t/e62woOWQ/swSmQs40xye7tSe7ybCDHWhuY\nxbDTHfk9YoyZUnJ0aJyV/wLoaa39q/u3ke56Tinx+NOA3wG/t9YWu7EPAq8BPwQauTF7rLWfAp8a\nY34BHHTv7+k5KOmbb/IqWhwxBQVHQ8a0b9+R+fMXV0E2oeXmHqjU/dLT67FvX+jn9MiRwkpvA+CO\nO37HqFFDKCys8L+XvLw8xo+/kzlzcqhRI3SRAygqCt1a/frrQ9SsWfn85XiHDh0+of3Bi/T0euVu\n4+uvy/28HLPOOONMbrzxt3TpciGHDhVy6FDV7o8VfXg50WOGGwI3rLWFwBagHZCOUwz+XiL+b+6y\nU3FGlR2BXGPMX3FGbu9ba719FfL32y3GaaP2d/90KdAAZ5INQAdgjDHmYOAHeASoBWSUscp27r9v\nBW3jf9baWwPFOOjvO4HlwFhjzGPGmI3A8+7iZJy250rgBWPMf4wxjwJNrLUfujEReQ4kPmRmtmHA\ngOs9xW7d+jZr18bftUt7977iMl5MAAAPTklEQVSM3r0v8zsNiSFJSUl07Hge9903g2XLnvTt2qOh\nnGgxPFLi92SgCCjvzTzQ2ztird0ENMUZxX0A3AS8a4xpU4k8lgHnuW3MAcBz1tp97rICnDZn+6Cf\nc4CWOCPKUI+pXMaYswEL/Ax4H6dFeqxPZK0tttb2xWnNPgQ0w2mD5rjLI/kcSBwYOnQkLVqc4Sn2\n4Yfj72T83bs/ZffuT/1OQ3xWt249LrroYsaMGceqVS8wd+5Cunf/sedOhx9OtE3aCfcUB/dcwE7A\nMmvtt8aYPUA34E9B8d2AvcA3xpi7gdettauB1caYW4DPgctxCkuwCntA1tqPjDGv4xSiq4Dga/r8\nE3ciTeAPxpjebmxZH9M/cP89D2ckizHmB8DHwNUlYrOA3dbaXwate5R7s4Yx5jxggLV2nPuYZhlj\nxuO0cG8M8zmQaiA1NZWJEyd5apcGZpeG0y4Viabk5GTS0tJITU3jpJNOon79+tSv35BTTjmF009v\nSkZGCzIyzqBZs+YkJcXsyQplOtFieJsxxgL/wJn80QAIHECaCjxojNkJ/BXoidMGvNtaW2yMyQAG\nGWNGAJ/gHG88GWcWaEkHANzi8q9yclmG03bMB14O+vtU4E/GmB04x+NaAYuAF621pS6PYK390Biz\nFsgxxtyAM9nlPpyJO28DPwkKzwUyjDE/BT4ELsYZhQLUdO9zkzHmO+BRoB5wRdBjDOc5kGoi0C71\nOrt07dpVXHVVfM4uFf9lZ99Dr15X+J1GzDvR0r0YmAi8gzMZpGdgRqe1dhHOpJMJuKdd4Jx6MdO9\n7xiciSYrcArJLcAQa+0GSluPc3xyMzCynFyewjm1YYW19tjMD/cUh0E47dMdOIVwOTCqrJW4snCO\nGa7FKUxpwC/KKJ5zgdXutt/DmfU6CjgEnG+t/QhnpPpTd3lgsk/g+GY4z4FUI0OGjAirXbp37xdR\nzkgksZ3oyPA9a+348ha6l0+bU86ygziFo8yiZK3NCLp9AOhRImRpifh9OJNiylrXCpyC44m7rqHu\nT8llk3BnqroTXbLKWMUzQfEvUc7VckI9B1J9paWlkZ19D6NHD/XULp0+fQpz5jysdqlIlMRXU1ek\nGmnd+mz69x/kKXbr1rfiYnbp5Zf/issv/5XfaYiELX7OqhaphoYOHcmmTX9j165/h4x9+OG5dOly\nEaed1rgKMqucyZPv8zsFkUqp9MjQWlvDWvuHSCYjkmjS0tKYONH7Vz1Nn65rl4pEg9qkIj5r3fps\n+vW7zlNsrLdLH3jgPh54QKNDiT8qhiIxYNiwUWRkeJtdmpMzL2Znl65c+UdWrvyj32mIhE3FUCQG\nOLNL7/bULs3LO8T06VOqICuRxKFiKBIj2rRpG1a7dM2a2G2XisQbFUORGOK0S1t4is3J0cn4IpGi\nYigSQwIn46tdKlK1VAxFYkybNm3p23dg6EBir13aqFEjGjVq5HcaImHTSfciMWj48NFs3ryRXbs+\nCRmbkzOX4uKikHFV4eWX/+p3CiKVopGhSAxKS0tjwgTv7dL8/PwqyEqk+lIxFIlRZ5/tvV0aKzZs\nWM+GDev9TkMkbGqTisSwcNqlsWDcuJsB2Lp1h8+ZiIRHI0ORGBZOu1REKk/FUCTGOe3SAX6nIVKt\nqRiKxIFhw0bTvHmG32mIVFsqhiJxoGbNmkyYcA9JSXrJikSDXlkicaJt23ZxN7tUJF5oNqlIHAnM\nLv30011+p1KmJ5981u8URCpFI0OROBLr7dKWLVvRsmUrv9MQCVtsvqJEpFyx3C4tKCigoKDA7zRE\nwqZiKBKHhg8fTbNmzf1Oo5SuXTvStWtHv9MQCZuKoUgcqlmzJtnZsdsuFYk3eiWJxKm2bc/h2mt1\nMr5IJGg2qZSrV68r6NChU4UxjRs3qaJspCwjRtzA5s0b2b37U79TEYlrNYqLi/3OQcKUm3tA/2kR\nlJ5ej9zcA36nUS106tQW0IW6I0X7ZmSlp9erUd4ytUlFRCThqU0qIhEzduxtfqcgUikqhiISMYMG\nZfmdgkilqE0qIiIJT8VQRCJm5MgsRo7M8jsNkbCpTSoiEbN16xa/UxCpFI0MRUQk4akYiohIwlMx\nFBGRhKdiKCIiCU8TaEQkYjp37up3CiKVomIoIhGTk/OI3ymIVIrapCIikvBUDEUkYh59dDGPPrrY\n7zREwqY2qYhETE7OXACGDRvpcyYi4dHIUEREEp6KoYiIJDwVQxERSXgqhiIikvBUDEVEJOHVKC4u\n9jsHERERX2lkKCIiCU/FUEREEp6KoYiIJDwVQxERSXgqhiIikvBUDEVEJOGpGIqISMJTMRTfGGOu\nM8YUB/2+yxhzl8f7tjHGXHaC219ijPnriawj2owxtY0xNwb9PskY87GfOSWaRN9Pgx+vMSbFGDM2\naFmWMeaoX7lFkoqhxJLzgQc9xq5146u7W4DxQb/PArr4lIs4Em0/DX68fTn+sT8FnF7lGUWBvs9Q\nYoa1NjeM8BpRSyS2HPc4rbUHgYM+5SIk3n5a4vGW3B/zgfyqzSg6VAylFLclNBoYAZwN/BMYZ639\nm7t8KVALOBXoCNxprV1gjBkB3AY0Az4GZllrlwWt98fATKANsB14scR2dwFLrLVT3d9/CUwC2gFf\nAQ9ba2e6LaMzgXuMMVnW2gxjTE1gGjAAqAO8A9xhrX0jaP03ufn9EHiWEJ0R93kYBmThfDr+FJht\nrV0cFBPqMXfG+STdAdiFM7JbArSw1u4yxmS4z0lP4GTgs6DHmQVMCcqlJ9ADuM5ae5YxZgPwibU2\nK2h71wJLgdOstd+Gyi+eaT/19jy4MUOBcW4+nwFzrLXz3WV1gPlAL5x9cDuQba19LfjxApuAx4O2\nOcRd/RJrbYoxZhnQ3FrbI2i7FwBvAi2ttR8bY64CJgMG5/WwBOc1VVTRY6wKapNKeWYCi3DexLcC\nLxtjzghafi2wGugMrDbG3ADcB0wE2gIPAA8ZYwYDGGPOwnlT2QS0BxYCd5a3cWNMV+AF4BU3/hac\nN5URwNU4L6Tf830LajlwiZvXecBrwHpjTCt3fYOA2ThvRB2APUB/D8/DAzhvFB2AjcACY0xzd52h\nHvPpwF9w3pw6Ar9zY4I9D5yEU+hau49jhjGmPU4L6gE318bA5hL3XQb0NsacFPS3AcBqtxBWmF81\nof00xPNgjBmHsw/PAc5xY2caY25173svTuH/ufvvO8Aat0gG2wz8xr3dGGf/DLYMuNjd7wMGAJvd\nQtgLeAJ4CKdojwd+i/O68J1GhlKeR6y1jwC4Ezh+ivPJc4K7fK+1dm4g2BgzEZhsrX3G/dNOt2hk\n47xIRgD/AW5xPwVaY8zZwO3lbH8MsNFaG3ihfOjmcdRa+7UxphA4aK3Ndd/ArgXaWmv/6cZPNsZ0\nA24FRuG8iP8QeEzAne4IIJTHrLUr3cd4OzAcuABnlBjqMY8E/guMttYWAh8YY04D5rnrq4UzinvS\nWvuZ+7fJ7nrbWWu3G2MOAoXW2r3u8uDcnnbX1Qt41hjTAPglcKW7PFR+1YH20wqeB2NMNk7RmWOt\nXeLGfuQWyjuMMbOBs4ADOF2G/caY24BVQGHwBqy1BcaY/e7tsvbH9TjFuy8w2xiT7N6e5C7PBnKs\ntY+5v+80xtQDHjHGTPF7dKhiKOXZELhhrS00xmzBaQMF/DtwwxiTjnMQfZYxJnjkkwKkGGPScD6F\nv1Nih3+D8rWjRHvKWru8nNgO7r9vlnhx1nR/cLdfsgC8gfNJuSIfBm1/n7v+NI+PuSPwtlsIAzYF\nrS/fGDMf6Ou2k1rijC6SgOQQeWGtPWCMWY0zcngWuAan+L7qJT9rbUGobcQB7aeO8p6HdJx2699L\nxP8Np0ieijNSfA7INcZsBv4MLLfWfhdim8ex1hYbYx7H2R9nA5cCDfh+BNkBON8dnQck4bSyMwj6\nv/KDiqGU50iJ35OB4DeI4IPmgTfVm4G/lrGuo0AxpScTVPRmXHL7FQmspyulD+Yfdv8Nd/sl7x+s\nBt4e81EqOBThtqE24bwOn8H5ZP0mzqjTq2XAc+4n7AHAE+6boZf8qgPtp2XnEXgeyitogQ9bR6y1\nm4wxTXHapD8DbgJuNcZ0t9a+72HbwZYBE40xLXH2x+estfvcZQXADJxWaUl7wtxOxOmYoZSnU+CG\nMSbF/f2dsgKttftxDspnWGs/DvwAPwFucz9lb8f5VBj8Aey8Crb/Qcnlxpipxpg17q/BX8QZaDn9\nsMT2bwF+5S7bDlxYYhsVbb9CHh/ze0Ant10U0Dnodg+ckWB3a+1ka+0qnEkVSXz/hhjqC0fX4YwG\nhwIX4xyT8ppfdaD91FHm82Ct/Ran0HQrEd8N2At8Y4y5G7jIWrvaWnsDTociGbi8jO1UuD9aaz8C\nXgf6AVdx/Cj3n7gTaYIeezucY7i+z7rVyFDKc5sxxgL/wDle0gBYXEH8VJzjBLtx3qA747RKZrjL\nF+EcX1lkjJmJ0/YZU8H6ZgFvG+dk3yeBc4GxQfc5ALQyxjRxD84/BSx2Z+J9iFMcRuN80g2s7xlj\nzFvASzjHbroR1LashFCPOQfnWFCOMeZBoBXu7FCcN5XAlPWBxpjncI7dzHb/FmibHQAaGKevVmrE\naK0tMsb8wV3vdmvtjjDyqw60n4Z+HqYCDxpjduKMiHvijI7vdlubGcAgd9LPJzjHG0/G6VKUdADA\nGHMe8K9yclmG03rNB14O+vtU4E/GmB04xyRb4TzfL1pry+rAVCmNDKU8i3EmYLyD8ybdMzDJoyzW\n2oU4kxZuB97H2fGn4Eyjxlr7H5xjCJnuOicC91ewvm04s/GuwflEOQNnunfg4PtsnMki7xljknAm\ntrwI/B+ww112tbV2nbu+NTinSNyEM2LrQsVvmiF5eMxf4kxuOR94F5iOUyABCqy1b7n3vQPnjeVh\n4A847dLA7MNVODMS3wPKu5LJMqAe7qjQa37VhPbTEM+DtXYRzuSVCW6O43BOvZjp3ncMzqzWFTgF\n+hZgiLV2A6Wtxzk+uRlnglhZngJSgRXW2mPteGvtn4FBOO3THTiFcDnOxCHf1SguDtWFkURjnHOI\nBllr/+B3LvHMGNMGONla+3rQ3/rhzCCtG/xGIeHTfurQ8xAZapOKRM+PgLXuOWyvAy1wRiBPqRCK\nxBa1SUWixFr7Ms6VRO7FaT+twGmR3VDR/USk6qlNKiIiCU8jQxERSXgqhiIikvBUDEVEJOGpGIqI\nSMJTMRQRkYT3/787ch+UQyRTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "mglearn.plots.plot_binary_confusion_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 各项指标计算公式\n",
    "###### 精度(accurary)\n",
    "###### 所谓accuracy精度即是正确预测的数量(TN+TP)除以所有样本数(TN+FP+FN+TP):\n",
    "\\begin{equation}\n",
    "\\text{AUC} = \\frac{\\text{TP} + \\text{TN}}{\\text{TP} + \\text{TN} + \\text{FP} + \\text{FN}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### 准确率(Precision)\n",
    "###### 准确率(precision)度量的是被预测为正例的样本中有多少时真正的正例\n",
    "\\begin{equation}\n",
    "\\text{Precision} = \\frac{\\text{TP}}{\\text{TP} + \\text{FP}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### 召回率(Recll)\n",
    "###### 召回率(recall) 正样本中有多少被预测为正类\n",
    "\\begin{equation}\n",
    "\\text{Recall} = \\frac{\\text{TP}}{\\text{TP} + \\text{FN}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### F 综合度量值\n",
    "\\begin{equation}\n",
    "\\text{F} = 2 \\cdot \\frac{\\text{precision} \\cdot \\text{recall}}{\\text{precision} + \\text{recall}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.970760233918\n",
      "[[ 62   1]\n",
      " [  4 104]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "  malignant       0.94      0.98      0.96        63\n",
      "     benign       0.99      0.96      0.98       108\n",
      "\n",
      "avg / total       0.97      0.97      0.97       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier #随机森林分类器\n",
    "from sklearn.model_selection import  train_test_split #数据集合拆分器\n",
    "from sklearn.datasets import  load_breast_cancer #数据集\n",
    "from sklearn.model_selection import KFold #k折交叉验证\n",
    "from sklearn.model_selection import GridSearchCV #估计器\n",
    "from sklearn.metrics import confusion_matrix #混淆矩阵\n",
    "from sklearn.metrics import classification_report #输出混淆矩阵各项指标\n",
    "cancer = load_breast_cancer()\n",
    "x_train,x_test,y_train,y_test = train_test_split(cancer.data,cancer.target,random_state=0,test_size=0.3)\n",
    "\n",
    "#构建参数组集\n",
    "param_grid ={\"n_estimators\": [x for x in range(20,100,10)],\"max_depth\":[y for y in range(2,10)]}\n",
    "\n",
    "#构建网络搜索分类器\n",
    "rf = RandomForestClassifier()\n",
    "kfold = KFold(n_splits=5)\n",
    "grid_search = GridSearchCV(rf,param_grid=param_grid, cv=kfold)\n",
    "\n",
    "#进行交叉验证训练\n",
    "model = grid_search.fit(x_train,y_train)\n",
    "print(model.score(x_test,y_test))\n",
    "\n",
    "#输出混淆矩阵及各项指标\n",
    "pred = model.predict(x_test)\n",
    "confusion = confusion_matrix(y_test,pred)\n",
    "print(confusion)\n",
    "\n",
    "#输出准确率,召回率,f1值,数量\n",
    "print(classification_report(y_test,pred,target_names = cancer.target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 准确率(precision)-召回率(recall)曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x120e5e128>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAFyCAYAAADLZb9oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzt3Xt0XWWd//F3bm2SNm1SmllYKDeR\nB5WrohaoIM6g0hF+ODKzBAa0CAoqF0EQRhREGRyQHyhOQUTKyPyEkQEUBiiOAnIXBuROH+lAkTuh\nTdM0adPm8vvjnJST0kt6TvZzTtL3a62unr2fffb5nm/brE+ffasaGBhAkiRJaVSXuwBJkqRNieFL\nkiQpIcOXJElSQoYvSZKkhAxfkiRJCRm+JEmSEqotdwHD1dbWOSrvidHS0kh7e3e5yxjV7GHp7GHp\n7GHp7OHIsI+lS9HD1tamqnWNOfOVsdramnKXMOrZw9LZw9LZw9LZw5FhH0tX7h4aviRJkhIyfEmS\nJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJChi9JkqSEDF+SJEkJGb4kSZISMnxJkiQllGn4CiF8\nJIRw11rWHxhCeDiE8EAI4Zgsa5AkSaokmYWvEMJpwBVA/Rrr64CLgE8A+wJfCiFsnlUdWVm2spNv\n3XMa75u7HV/5nflRkiQNT9XAwEAmOw4hfBZ4Arg6xjijYP0uwPkxxk/lly8C7o8xXre+/fX29g2U\n+0GYg26KN/HVW7/Ky0tfXr1u8WmLaWloKWNVkiSpglSta6A2q0+MMV4fQthmLUOTgI6C5U5g8ob2\n197ePUKVFe+Nrtc5455T+a/nfzNk/eHvPZLeZbW0Let8x3taW5toa3vneg2fPSydPSydPSydPRwZ\n9rF0KXrY2tq0zrHMwtd6LAUKK2oClpShjmHr6+/jqqd/zj//8Rw6Vy5dvX5qw1S+P/Nf+Mz2h5Sx\nOkmSNJqUI3w9C7wnhDAFWAbsA/ywDHUMy5NvPcGpd53Io28+MmT9YTsewVl7fY+W+illqkySJI1G\nycJXCOEwYGKM8fIQwsnA7eRO+L8yxvhKqjqGa9mqZVzw0Hlc/sQc+gb6Vq9/d/P2XLDvxczcYp8y\nVidJkkarTMNXjHEhMCP/+pcF628Gbs7ys4s1MDDALc/fzJn3fpNXu97OhOOqx3HiB0/h+N2/Tn1t\n/Xr2IEnaWN0PLWPx3DaW3dFBX0cfNZNrmPjxyUyZ3UrjhyeWuzxpRJXjsGPFWtjxAv90z6n87i+/\nHbJ+5hb7cP4+F7F9y3vKVJkkjU0DqwZ49RsvsuSaRUPW97X30XH9YjquX0zzYZsx7YKtqapb58Vj\n0qhi+Mq75tl/55t3n8yKvhVD1r9rwjR2nrorNzx3HY11E2iobWBC3QQaaxtprGuksXbC0N/rGmmo\nbaS+pp6qKn9QSNL6rC14rWnJL3PjW1y8TYKKpOwZvsgdajzt7q/T09fzjrHXul7l0scv2eh9VldV\n01DbyMRxE6ivaSgIZxOYUNtIQ20DjatDXEGYK1jXUNswJNRNKAh6NdWVcc8zSSpW90PLhgSvplnN\ntJ64OfU7NbLiqW7afvQ6nbfmLoZf8stFtBw+lcYPeQhSo5/hK2/PaXtz10t3jNj++gf66Vq1jK5V\ny0Zsn4XG14ynsTY3yzY0tBXOxhWEuPz42mbtGgqWx9XUUbXu+8KVRX1P7okCKp49LJ09LN2aPXzr\n56+tft3wqYm0/LSVvqp+ugaWwfuh5aet9B7Ty/J5uZ+jb17xKlN3fVfyuitNYR/rasYxvmZ8mSvS\nxjJ8AVVVVVz76Rt4dtEzdPQsobu3i+5V3XT3dtO1qovu3m66V+XWLe/tzi93D9lucLy7t4vlvcvX\nOos2knr6eujp66G9pz3Tz5GkrPx63q+ZnL/H9uen/yPx5/Ed2+y45Y5cyqUAvDrvZT58xR5Ja6x0\n46rH8c2PnMnxu59U7lK0EQxfedVV1bx/6k4jtr/e/l6W93bTMKmav7zxRi6YDQlsuZD2dmjLB7gh\nvw9uv/wdQW+AbB4LJUmpTFzx9iHEBZsvWOs2hesLt1fOyv6VXPPs1YavUcbwlZHa6lqaxk2itamJ\n2hH+gTEwMMCKvhVDw9yqfJhbz6zd4Os1tyuctVvV3zuitY6EqirI6BGkmwx7WDp7WLo1e9hV38Wk\n5ZMA2PmtXXhuy+fe8Z4dXn/7KvNlDV1MqDOAVVXBit4V9OZ/Xu/QsmOZK9LGMnyNQlVVVTTUNtBQ\n28BmbFbucjLnc8xKZw9LZw9Lt2YPX370BTquXwzAFS9eyfSztxtylfjAwAAvzX6ezvwT6LY6YBte\nOObVtEVXoNbWJj551QH89sV5AOw7fb8yV6SNVV3uAiRJm6Yps1tXv+68dQkvzX6e7ke76F/ZT/ej\nXbngdevbj/6dclTr2nazyVnZt5J7X7ln9bLha/Rx5kuSVBaNH55I86Gbrb7dROetS4aErULNh23m\nbSby7n/pfrp7uwCoq67jyicvL3NFo0tNVS1HfPBQtq8fufO8N5bhS5JUNtN+uDVUvX0j1bUZvMO9\ncu5+8e7Vr1f1r+LyJy4tYzWj09ynf8bjR86npX5KWT7f8CVJKpuquiq2uHgbWg6fyuIr21h2Zwd9\nS/qoaa5h4n6TmXJUqzNea5jWNK3cJYx6K3pX8ELH84YvSdKmq/FDEw1Zw3TU7kcxrncif1m6sNyl\njDqXPf6vvLLs5XKXYfiSJGk0qa6q5oBt/7bcZYxKNzx3XUWEL692lCRJSsjwJUmSlJDhS5IkKSHD\nlyRJUkKGL0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJChi9J\nkqSEDF+SJEkJGb4kSZISMnxJkiQlZPiSJElKyPAlSZKUkOFLkiQpIcOXJElSQoYvSZKkhAxfkiRJ\nCRm+JEmSEjJ8SZIkJWT4kiRJSsjwJUmSlJDhS5IkKSHDlyRJUkKGL0mSpIQMX5IkSQkZviRJkhIy\nfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJChi9JkqSEarPacQihGpgD7Ar0AEfHGBcUjH8D\nOBToB/45xnhjVrVIkiRViixnvg4G6mOMewKnAxcODoQQmoETgD2BTwAXZ1iHJElSxcgyfM0E5gHE\nGB8E9igY6wJeBCbkf/VnWIckSVLFyOywIzAJ6ChY7gsh1MYYe/PLLwHPADXAeRvaWUtLI7W1NSNf\nZQKtrU3lLmHUs4els4els4els4cjwz4WpzBHtLRMKFsfswxfS4HCb1VdELwOAN4FbJtfvj2EcF+M\n8aF17ay9vTubKjPW2tpEW1tnucsY1exh6exh6exh6ezhyLCPxevt7Vv9ur29i7Zx2fVxfcEuy8OO\n9wGzAEIIM4AnC8bageVAT4xxBbAEaM6wFkmSpIqQ5czXjcD+IYT7gSpgdgjhZGBBjPGmEMLfAA+G\nEPqBe4H/zrAWSZKkipBZ+Iox9gPHrrF6fsH4WcBZWX2+JElSJfImq5IkSQkZviRJkhIyfEmSJCVk\n+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJChi9JkqSEDF+SJEkJGb4kSZISMnxJkiQlZPiSJElKyPAl\nSZKUkOFLkiQpIcOXJElSQoYvSZKkhAxfkiRJCRm+JEmSEjJ8SZIkJWT4kiRJSsjwJUmSlJDhS5Ik\nKSHDlyRJUkKGL0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJC\nhi9JkqSEDF+SJEkJGb4kSZISMnxJkiQlZPiSJElKyPAlSZKUkOFLkiQpIcOXJElSQoYvSZKkhAxf\nkiRJCRm+JEmSEjJ8SZIkJWT4kiRJSsjwJUmSlJDhS5IkKSHDlyRJUkKGL0mSpIQMX5IkSQkZviRJ\nkhIyfEmSJCVUm9WOQwjVwBxgV6AHODrGuKBg/ADgrPzio8BXY4wDWdUjSZJUCbKc+ToYqI8x7gmc\nDlw4OBBCaAIuAD4dY5wBLASmZliLJElSRcgyfM0E5gHEGB8E9igY2wt4ErgwhHAP8EaMsS3DWiRJ\nkipCZocdgUlAR8FyXwihNsbYS26Waz9gN2AZcE8I4YEY45/XtbOWlkZqa2syLDc7ra1N5S5h1LOH\npbOHpbOHpbOHI8M+FqcwR7S0TChbH7MMX0uBwm9VnQ9eAIuAh2OMrwOEEO4mF8TWGb7a27uzqjNT\nra1NtLV1lruMUc0els4els4els4ejgz7WLze3r7Vr9vbu2gbl10f1xfssjzseB8wCyCEMIPcYcZB\njwA7hRCmhhBqgRnAMxnWIkmSVBGynPm6Edg/hHA/UAXMDiGcDCyIMd4UQjgDuD2/7a9ijE9lWIsk\nSVJFyCx8xRj7gWPXWD2/YPxa4NqsPl+SJKkSeZNVSZKkhAxfkiRJCRm+JEmSEjJ8SZIkJWT4kiRJ\nSsjwJUmSlJDhS5IkKSHDlyRJUkKGL0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUqodn2DIYR+\nYACoWsvwQIyxJpOqJEmSxqj1hq8YozNjkiRJI2hDM1/fWd94jPGckS1HkiRpbFtv+GLthxslSZJU\npA0ddvzu2taHEKqAbTOpSJIkaQzb0MwXACGELwE/BCYUrH4B2D6LoiRJksaq4Z5QfwawK3At8G7g\neOCPWRUlSZI0Vg03fL0ZY3wBeBLYOcY4h1wYkyRJ0kYYbvjqCiHsBzwBHBhC2BxoyK4sSZKksWm4\n4esE4EBgHrAZMB+4JKuiJEmSxqphha8Y41PA1THGfuAo4LMxxoszrUySJGkMGlb4CiH8APiX/GIj\n8O0QwtlZFSVJkjRWDfew46eBAwBijK8BfwN8NquiJEmSxqrhhq9ahp5gP47cA7clSZK0EYZ1k1Xg\np8AjIYSbyYWuWcBPMqtKkiRpjBruCfcXAYcDrwEvAf8YY7w0y8IkSZLGouEedgTYAZgC/BzYOZty\nJEmSxraNudpxFvB3QA0wO4RwYZaFSZIkjUXDnfn6JHAEsCLGuBTYn/zVj5IkSRq+4Yav/vzvg1c4\nji9YJ0mSpGEabvj6FfAfwJQQwknAPcAvM6tKkiRpjNrgrSZCCAG4GngMeBGYDvxf4KPZliZJkjT2\nrHfmK/8IoUeAPwP9McZTgdeBOcDWmVcnSZI0xmxo5utI4D3ANOCcEMLJwBbA38cYb8+6OEmSpLFm\nQ+GrM/8sx9dCCB8GfgF8OsbYl31pkiRJY8+GwlfhFY1vxRhPybIYSZKksW5DVzsWPjx7eZaFSJIk\nbQo2NPP1/hDC8/nXWxS8rgIGYozbZVeaJEnS2LOh8LVDkiokSZI2EesNXzHGF1MVIkmStCkY7h3u\nJUmSNAIMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJCG7rDfdFCCNXA\nHGBXoAc4Osa4YC3b3AL8JsZ4WVa1SJIkVYosZ74OBupjjHsCpwMXrmWb7wNTMqxBkiSpomQZvmYC\n8wBijA8CexQOhhAOAfqB2zKsQZIkqaJkdtgRmAR0FCz3hRBqY4y9IYSdgMOAQ4DvDGdnLS2N1NbW\nZFBm9lpbm8pdwqhnD0tnD0tnD0tnD0eGfSxOYY5oaZlQtj5mGb6WAoXfqjrG2Jt/fSSwBXAHsA2w\nMoSwMMY4b107a2/vzqrOTLW2NtHW1lnuMkY1e1g6e1g6e1g6ezgy7GPxenv7Vr9ub++ibVx2fVxf\nsMsyfN0HHAj8KoQwA3hycCDGeNrg6xDC2cDr6wtekiRJY0WW4etGYP8Qwv1AFTA7hHAysCDGeFOG\nnytJklSxMgtfMcZ+4Ng1Vs9fy3ZnZ1WDJElSpfEmq5IkSQkZviRJkhIyfEmSJCVk+JIkSUrI8CVJ\nkpSQ4UuSJCkhw5ckSVJChi9JkqSEDF+SJEkJGb4kSZISMnxJkiQlZPiSJElKyPAlSZKUkOFLkiQp\nIcOXJElSQoYvSZKkhAxfkiRJCRm+JEmSEjJ8SZIkJWT4kiRJSsjwJUmSlJDhS5IkKSHDlyRJUkKG\nL0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJChi9JkqSEDF+S\nJEkJGb4kSZISMnxJkiQlZPiSJElKyPAlSZKUkOFLkiQpIcOXJElSQoYvSZKkhAxfkiRJCRm+JEmS\nEjJ8SZIkJWT4kiRJSsjwJUmSlJDhS5IkKSHDlyRJUkKGL0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk\n+JIkSUqoNqsdhxCqgTnArkAPcHSMcUHB+NeBz+UXb40xfjerWiRJkipFljNfBwP1McY9gdOBCwcH\nQgjbAYcDewF7Ap8IIeySYS2SJEkVIcvwNROYBxBjfBDYo2DsJeBTMca+GGM/UAesyLAWSZKkipDZ\nYUdgEtBRsNwXQqiNMfbGGFcBb4UQqoALgD/FGP+8vp21tDRSW1uTYbnZaW1tKncJo549LJ09LJ09\nLJ09HBn2sTiFOaKlZULZ+phl+FoKFH6r6hhj7+BCCKEeuBLoBL6yoZ21t3ePeIEptLY20dbWWe4y\nRjV7WDp7WDp7WDp7ODLsY/F6e/tWv25v76JtXHZ9XF+wy/Kw433ALIAQwgzgycGB/IzXb4DHY4xf\njjH2rX0XkiRJY0uWM183AvuHEO4HqoDZIYSTgQVADbAvMD6EcEB++zNijA9kWI8kSVLZZRa+8ifS\nH7vG6vkFr+uz+mxJkqRK5U1WJUmSEjJ8SZIkJWT4kiRJSsjwJUmSlJDhS5IkKSHDlyRJUkKGL0mS\npIQMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJChi9JkqSEDF+SJEkJ\nGb4kSZISMnxJkiQlZPiSJElKyPAlSZKUkOFLkiQpIcOXJElSQoYvSZKkhAxfkiRJCRm+JEmSEjJ8\nSZIkJWT4kiRJSsjwJUmSlJDhS5IkKSHDlyRJUkKGL0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk+JIk\nSUrI8CVJkpSQ4UuSJCkhw9cIuuGG6/jCFw7j97//bblLkSRJFaq23AWMJX/4w518+9vn8O53b1/u\nUiRJUoUaM+FrzmOXcMHD59G1atmI7XNC3URO/dAZfGW349e5za233swtt9zEyy+/RHd3Nz/4wTl8\n97vnMW3aFiNWhyRJGjvGzGHHSx+7ZESDF0DXqmVc+tglG9yuqamJ3/xmHiHsyJlnnmPwkiRJ6zRm\nwtdxux3PhLqJI7rPCXUTOW49s16Dttpq6xH9XEmSNHaNmcOOX9nt+PUeHsxSVdWYybCSJCljpgZJ\nkqSExszMV7nMmnXg6tc/+cnlZaxEkiSNBs58SZIkJWT4kiRJSsjwJUmSlNAmc85X90PLWDy3jWV3\ndNDX0UfN5BomfnwyU2a30vjhkb1FhSRJ0rqM+fA1sGqAV7/xIkuuWTRkfV97Hx3XL6bj+sU0H7YZ\n0y7Ymqq6qjJVKUmSNhVjPnytLXitackvc+NbXLxNgookSdKmLLNzvkII1SGEy0IID4QQ7gohbL/G\n+DEhhP8JITwYQvh0FjV0P7RsSPBqmtXMdrfvyPte+QDb3b4jTbOaV48t+eUiuh/euMcT9fT0cPPN\nvwbg5z//Kb/+9X+OTOHAQQd9ctjbHnLIgfT09AxZ9+CD93PuuWev9313330XRxzxD1x33bXFlChJ\nkoqQ5Qn3BwP1McY9gdOBCwcHQgibAycAewOfBM4LIYwf6QIWz21b/bppVjPT525Hw+4TqKqromH3\nCUyfu92QALb4yra17Wbd+1+8aHX4Go3uv/8evvzlr/H3f/+5cpciSdImI8vDjjOBeQAxxgdDCHsU\njH0YuC/G2AP0hBAWALsAD49kAcvu6Fj9uvXEzamqGnpOV1VVFVNP2JzOW5fktr+zg43xi19cycKF\nLzB37s8AuOeeu7nzzt/T0dHB0Ucfy8yZ+7Dffvux5ZZbsfXW2/K5zx3O+ef/MytX9jBu3HhOO+2f\naG5u4TvfOZ2uri56elZw3HEn8IEP7MHKlSs5++xv8cYbrzN58mS+//3zWb58Od/73rfp6uqir6+P\nY445jg9+8EOr61m48AXOO+8c6usbaGiop6lp0pB6H330f7j00kuoq6vjwAMP5v777+XZZ5+mubmZ\nnXbaZaO+uyRJKk6W4WsSUJhm+kIItTHG3rWMdQKT17ezlpZGamtrNqqApzv6Vr/ecr+/orrunRN9\n/ftN4AXm5wpc0kdra9Ow93/SScfzl7+8wGmnncwll1zC9OnTOPfcc/njH//IFVdcwWc+87e89tpr\n3HDDDbS0tHDSSSfxxS9+gX333ZcHHniAuXMv49hjj2Xp0iVcddVVLFq0iIULF9La2sSKFcs544zT\n2HLLLTniiCNoa3uJ2267jY99bB8+//nP88Ybb3DooYfyu9/9jpqaalpbmzj//Cs45ZSvs/fee3P5\n5Zfz/PPPD/k+zc2N9Pf3ct11NwDwzDOPM2vWLPbZZ++N6ms5bMyfi9bOHpbOHpbOHo4M+1icvbfe\niz+9+ShN45rY6z170DS+PH3MMnwtBQq/VXU+eK1trAlYsr6dtbd3b3QBNZNr6GvPBbCX73yTht0n\nvGOb7ke73t6+uYa2ts5h73/x4i5Wreqjra2Trq4ett56e9raOqmpaWTp0mW0tXXS0tJCb28tbW2d\nPPvsfF55ZQ5z5lwGQG1tLc3Nm3PQQZ/la187gd7eXg455HO0tXXS1DSJ8eMn519P5rXXFvHss5GZ\nMz9OW1sn1dWN1Nc38txzf6Gvr5+2tk7mz49Mm7YdbW2dbLfde3nmmTjk+yxZ0s20adNXr1uxYhUd\nHcs36juXQ2trU8XXWOnsYensYens4ciwj8X7xm5n8v7Ju7FfmMmKpbCC7Pq4voCcZfi6DzgQ+FUI\nYQbwZMHYQ8C5IYR6YDzwXuCpkS5g4scn03H9YgDafvQ60+duN+TQ48DAAG/9+PW3t99vvZNv71BV\nVc3AQH/B8ju3qa5+e7Ztq6224dBD/5Gdd96VF19cyJ/+9Aj/+78L6O7u4oILfsRbb73Fcccdxd57\nf/Qdh0gBtt56Wx5//DF22GFH2trepLNzKZMmTR6y/6eeeoIZM/Zi/vyn11pzdbW305AkbZoa6xr5\n7A7/QOvU8gbYLMPXjcD+IYT7gSpgdgjhZGBBjPGmEMKPgXvInfT/rRjjipEuYMrs1tXhq/PWJbw0\n+3mmnrA59Ts1sOKp5bz149dXn+8FMOWo1o3af0tLC6tW9TJnzo8ZP37D1wt89asncuGFP2DlypX0\n9KzgxBO/wZZbTmfu3MuZN+8Wamvr+OIXv7zO9x955GzOO+8c7rrr9/T09HDaad+itvbtP8JTTjmd\ns846g2uuuZrm5mbGjRvxaxgkSVKJqgYGBspdw7C0tXUWVegrJy7c4H2+AJoP2yyT+3w5PVw6e1g6\ne1g6e1g6ezgy7GPpUvSwtbVpnYeaxvyzHaf9cGuaD9tsvdsM3uFekiQpa2P+DvdVdVVscfE2tBw+\nlcVXtrHszg76lvRR01zDxP0mM+WoVho/5LMdJUlSGmM+fA1q/NBEQ5YkSSq7MX/YUZIkqZIYviRJ\nkhIyfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJCo+bxQpIkSWOBM1+SJEkJGb4kSZISMnxJ\nkiQlZPiSJElKyPAlSZKUkOFLkiQpodpyFzBahRCqgTnArkAPcHSMcUHB+DeBQ4GlwPkxxv8qGNsH\n+H8xxulpq648xfQxhDAF+DPwVH6zG2OMP0pbeeUosocTgEuBbYFxwPExxoeSF18hiuzhxcBu+U02\nB5bEGGekrbxyFNnDrYCrgSpgMXBYjLE7efEVosgebgv8G7kevgh8aVPu4aAQwkeAf4kxfmyN9QcC\n3wF6gStjjD8LITQA/w78FdAJfD7G2JZlfc58Fe9goD7GuCdwOnDh4EAIYWfgMGAG8AngnBBCY35s\nOnAKUJe84spUTB8/AFwTY/xY/tcmG7zyiunhqcBTMcaPAscAIXnVlWWjexhjPCn/g31/oINcHzdl\nxfw9/DrwHzHGfYCngS8mr7qyFNPDC4DL8v+W7wJOTl10pQkhnAZcAdSvsb4OuIhc//YFvhRC2Bw4\nDngy38NfAGdmXaPhq3gzgXkAMcYHgT0Kxt4L3BVjXBFjXAE8B+wSQqgHLgO+krrYCrbRfQQ+CHwg\nhPCHEMJ1IYR3pS66whTTw08CK0MItwPfBm5PW3LFKaaHg44HfhtjfDJVsRWqmB4+BrTkt5kErEpX\nbkUqpofvA27Lb3Nffh+buv8F/m4t698LLIgxtscYVwL3Ah+loO/kevk3WRdo+CreJHL/2x3UF0IY\nPIz7JLBPCKEphLAZsBcwAfgJ8MMY4ytpS61oxfRxPnBWjHFf4NfAJSkLrkDF9HAq0BJj/CRwM/DD\nlAVXoGJ6SAhhHPBl7B8U18OXga+FEJ4GDgCuS1lwBSqmh48BB+W3OSi/bpMWY7yetQf5NfvbCUxe\nY/3gukwZvoq3FGgqWK6OMfYCxBifJRe0biM3bfxHcseXPwqcFUK4C5gSQrg2acWVaWP7+BZwB3Bn\nfvsbgd2TVVuZiunhIuCm/PY3M/R/2JuiYnoIuf8h3x1jLPyBvqkqpocXAF+IMb4fOJHcIZ9NWTE9\nPAU4KIQwD+jn7b+beqc1+9sELFlj/eC6TBm+incfMAsghDCD3P9KyC+3AlNjjDPJ/UCZDtwbYwyD\n5ykBi2OMn0tfdsXZ2D4+Re5Y/mfzm/018EjKgitQMT28d/A9wOD5NpuyYnoIufB1G4LietjO2zMO\nr/L2IchNVTE93B/4bozxU+TC13+nLnoUeRZ4TwhhSn7Weh/gAQr6Tm4G9p6sC/HB2kUquCplF3JX\nmcwm94e3gNxMwmXkTgxfCZwRY7x7jfe/HmPcPGnRFaiYPuav7rkyv30XuSuCXitD+RWhyB5OIRdi\n30Vuev7IGOPC9NVXhmL/PYcQbgG+FWN8rBx1V5Ii/x6+j9xsTk3+PSfGGP9UhvIrQpE9/Ajwr+Su\njnwa+GqMcVM/d44QwjbAtTFbOlD9AAACs0lEQVTGGSGEw4CJMcbLC652rCZ3teO/5i9c+DdyPw9X\nkrvq9vUs6zN8SZIkJeRhR0mSpIQMX5IkSQkZviRJkhIyfEmSJCVk+JIkSUrIB2tLGhXyl47/GXgG\nGCD3QPBXgdkxxpdL2O+xADHGy9Yxfg7wPzHGm9Y2Lkkby1tNSBoV8uHrrhjjNgXrLgSmxRgPLVdd\nkrSxnPmSNJrdCZwXQlhI7nEru5F7jNengJPInVrxCLkbT67I32zxTHIzZw8DxwDfyu/rXHI3790p\nvzwnxvizEMJV5ELfVSGE2eQe5zKQ3+/XYozLQgivAf9J7gG9vcA/xBhfyPKLSxq9POdL0qgUQqgD\nDiH3eBCA22KMAWglF6r2ijHuBrwJfCOEsAVwEfCJ/LMEa4C/LdjlXsCUGOPu+fUfXePzdiYX1PaN\nMe5M7ukKZ+WHNwd+n3/v3cDXRvr7Sho7nPmSNJpMCyEMPspnPPAQcDrwCXIzXwD7Ae8BHgwhQO7c\nsEeBPYH7Bs8PizEeARBC2C3/vqdyi+F24Fbg1DU+e1/g5hjjovzy5cDcgvF5BfvZp7SvKWksM3xJ\nGk1ezc9mDZEPWcvzizXAr2KMJ+THJpL7WbcvucOFg+9pLdxHjHFRCOH95B5UPAt4NL88aM0jBVUU\n/AyNMa7IvxzIj0nSWnnYUdJYcxfwmRDCX4UQqoBLyZ3/9TAwI4Qw+ED7i4D/M/imEMJBwNXALcAJ\nwDJg+hr7PSj/UHLIHdq8M7uvIWmsMnxJGlNijI8D3wXuAJ4mNxP2gxjjq8CJwO0hhKfIzZQVHja8\nLb/uaXKHM/89xvhkwX6fAM4D/hBCmA80kzt5X5I2ireakCRJSsiZL0mSpIQMX5IkSQkZviRJkhIy\nfEmSJCVk+JIkSUrI8CVJkpSQ4UuSJCkhw5ckSVJC/x9icMaXIqrX5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precision,recall,thresholds = precision_recall_curve(y_test,model.predict_proba(x_test)[:,1]) #直接使用上文中的model\n",
    "#找到最接近于0的阈值\n",
    "close_zero = np.argmin(np.abs(thresholds-0.5))\n",
    "\n",
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(precision,recall,label=\"rf\",c=\"g\",lw=3)\n",
    "plt.plot(precision[close_zero],recall[close_zero],\"o\",c=\"m\",markersize=10,label=\"threshold rf\", fillstyle=\"none\",mew=3)\n",
    "plt.xlabel(\"Precision\")\n",
    "plt.ylabel(\"Recall\")\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ROC与AUC,ROC即是受试者工作特征曲线,AUC表示ROC曲面下的面积大小,介于0~1之间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC分数: 0.995664315109\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsAAAAFkCAYAAAAuSo2KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAFh1JREFUeJzt3X+s3Xd93/HX/ZHgOLmJ7eSWlChN\nCIUPVFmTVgk4NEBbEVqyIjLKphG0aenCYEWjW8T4sUlDTBXVgLQpm6KJVXTaWlrRRRG/k1WiDOKQ\nkkG7JYN8qFmSsUKYQ67t6/hHEvvuj3ttLpbvj/jee4593o+HFOmc8/3qnLfylp1nvv76nrG5ubkA\nAEAV48MeAAAABkkAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClTA76A3ftmh3az13bunVzZmb2D+vj\nGQA7rsGea7Dn0WfHNQxzz9PTU2Mner3UFeDJyYlhj8AGs+Ma7LkGex59dlzDqbjnUgEMAAACGACA\nUgQwAAClCGAAAEoRwAAAlCKAAQAoRQADAFCKAAYAoBQBDABAKQIYAIBSVhXArbWXtda+eILXX9da\nu7+19pXW2lvWfToAAFhnkyud0Fp7V5K/l+TJ414/I8nvJLl64diO1tqne++PbcSgrJ/Hdx/IAw8/\nkczNDXuUdXfO1Kbsmz047DHYYPZcgz2PPjuu4ScvOT8Xn3/WsMf4ESsGcJJvJ3lDkv983OsvSbKz\n9z6TJK21e5K8IsmfLPdmW7duzuTkxEmMuj6mp6eG9tmnio9+5hu570H/nwIAbLyJ8bF84gN/M2ee\nMbz+O96KAdx7v6O1dukJDp2bZM+i57NJzlvp/WZm9q96uPU2PT2VXbtmh/b5p4rZfYeSJG953U9l\nYnxsyNOsr3Onzsre2QPDHoMNZs812PPos+MafvKSbdmzezj9t9SFz9VcAV7K3iSL33Uqye41vB8D\ndvWLfyyTE6P19yD9T04N9lyDPY8+O67hVNzzWgL4m0le2FrblmRfklcm+fC6TAUAABvkWQdwa+3G\nJOf03j/aWrslyd2Z/2kSH+u9//V6DwgAAOtpVQHce38kyfaFxx9f9Pqnk3x6QyYDAIANMFo3gAIA\nwAoEMAAApQhgAABKEcAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUp71VyFz+pqbm8vXv7Ur\n39n15LBHAQAYGgFcxDceeSJ3/Ldv5+HvzWZ8bCyvufriTE74AwAAoB4BPOIe/t7e/JcvfjvffHQm\nSXL1i38sf+uVl+XCbZuHPBkAwHAI4BH13cefzJ1f+t/52rd2JUkuf/62vOFVl+XSC88d8mQAAMMl\ngEfMD/YczCfveTg7Hvxe5uaSFzzv3Pzqq16QF1+yddijAQCcEgTwIp//80fzyXseTuaGPcnJe/rw\nkczNJRddcHbe8MrLcuULL8jY2NiwxwIAOGUI4EX6/9mdp54+kkueO5Xx0/Tvh50xMZ5XXvm8bP+p\nCzM+LnwBAI4ngE/g3W/+mWw6078aAIBRdJpe5wQAgJMjgAEAKEUAAwBQigAGAKAUAQwAQCkCGACA\nUgQwAACllPhhtzOzh/IH/7XnSJKnnjq85Hnf+X/7BjcUAABDUSKAv/noE/mLv3p8VededMHZOXNy\nYoMnAgBgWEoE8FFvf+MVufKyrcueMz42lrExXyEMADCqSgXw+PhYJsbd9gwAUJkaBACgFAEMAEAp\nAhgAgFIEMAAApQhgAABKEcAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAAAEoR\nwAAAlCKAAQAoRQADAFCKAAYAoBQBDABAKQIYAIBSJlc6obU2nuT2JFckOZTk5t77zkXH353kTUn2\nJvlg7/0zGzQrAACs2WquAN+QZFPv/Zok70ly69EDrbW/keTGJNuTvCbJv26tbd6IQQEAYD2sJoCv\nTXJXkvTe70ty1aJjL0nyxd77wd77wSR/leSn131KAABYJyveApHk3CR7Fj0/3Fqb7L0/k+SBJO9t\nrU0lOTPJy5N8dLk327p1cyYnJ0523pMyNbX72OPp6amBfjaDZ8c12HMN9jz67LiGU23PqwngvUkW\nTz2+EL/pvX+ztfbvknw+yc4kf57k8eXebGZm/0mOevJmZw8ce7xr1+zAP5/BmZ6esuMC7LkGex59\ndlzDMPe8VHiv5haIHUmuT5LW2vbMX/XNwvPpJBf03q9N8htJLk7y4FqHBQCAjbKaK8B3JrmutXZv\nkrEkN7XWbsn8Fd9PJ7mstXZ/kqeS/PPe++ENmxYAANZoxQDuvR9J8rbjXn5o0eO3rutEAACwgXwR\nBgAApQhgAABKEcAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAAAEoRwAAAlCKA\nAQAoRQADAFCKAAYAoBQBDABAKQIYAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEM\nAEApAhgAgFIEMAAApQhgAABKEcAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAA\nAEoRwAAAlCKAAQAoRQADAFCKAAYAoBQBDABAKQIYAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMA\nUIoABgCgFAEMAEApkyud0FobT3J7kiuSHEpyc+9956Lj70zypiRHknyg937nBs0KAABrtporwDck\n2dR7vybJe5LcevRAa21LknckuSbJa5LcthFDAgDAellNAF+b5K4k6b3fl+SqRceeTPJokrMX/jmy\n3gMCAMB6WvEWiCTnJtmz6Pnh1tpk7/2ZheffSfKNJBNJfmulN9u6dXMmJyee9aBrMTW1+9jj6emp\ngX42g2fHNdhzDfY8+uy4hlNtz6sJ4L1JFk89vih+X5vkx5M8f+H53a21Hb33ry71ZjMz+09q0LWY\nnT1w7PGuXbMD/3wGZ3p6yo4LsOca7Hn02XENw9zzUuG9mlsgdiS5Pklaa9uTPLDo2EySA0kO9d4P\nJtmdZMuaJgUAgA20mivAdya5rrV2b5KxJDe11m5JsrP3/qnW2quT3NdaO5LkniR/unHjAgDA2qwY\nwL33I0nedtzLDy06/r4k71vnuQAAYEP4IgwAAEoRwAAAlCKAAQAoRQADAFCKAAYAoBQBDABAKSMf\nwPu/ui9nfWAmv377VM54+cN5qP1l/u8/fjj7v7pv2KMBADAEq/kijNPS3NNz+e47H83uP/pBzkxy\ntPUPzxzOnjueyJ47nsiWG8/P8z50ScbOGBvmqAAADNDIBvDR+F3O7o/PH7/otksHMBEAAKeCkbwF\nYv9X9/1I/D597ab8wY378tQ9l+ayu1+cqeu3HDu2++M/yP773Q4BAFDFSAbwE7+/69jjqeu3ZP/7\nt+b7Fx5OJsdy1s+cnYt//7IfieAnPrbrRG8DAMAIGskA3veFPcceT//GhcnYj97jOzY2lgveceEP\nz/+zPQEAoIaRDODDew4fe7zp8s0nPGfT5Wf98Pzdh094DgAAo2ckA3jivIljjw8+uP+E5xx88MAP\nz98yccJzAAAYPSMZwOf84nnHHu/63ceSubkfOT43N5fHP/LYD8//hfMCAEANI/lj0LbdNJ09dzyR\nJJn93O5s3nswF148kTw9l/1ffzKPf+SxzH5u9w/P/7XpYY0KAMCAjWQAb37pOdnypvOP/Si0M+45\nmDfnnOSPHsnDx5275cbzs/nqcwY+IwAAwzGSt0AkyfM+fEm23Hj+succ/SY4AADqGNkAHjtjLBfd\ndmme/9mWp159Vg5sOpK58WRi20TO+9Vtef5nWy667VJfgwwAUMxI3gKx2Oarz8mBf7E1v/eZx/JP\n/s6VufyybcMeCQCAIRrZK8AAAHAiAhgAgFIEMAAApQhgAABKEcAAAJQigAEAKEUAAwBQigAGAKAU\nAQwAQCkCGACAUgQwAAClCGAAAEoRwAAAlCKAAQAoRQADAFCKAAYAoBQBDABAKQIYAIBSBDAAAKUI\nYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEMAEApAhgAgFIEMAAApUyudEJrbTzJ7UmuSHIoyc29\n950Lx65Mctui07cnuaH3ftcGzAoAAGu2YgAnuSHJpt77Na217UluTfL6JOm9/2WSn0+S1trfTvJd\n8QsAwKlsNbdAXJvkriTpvd+X5KrjT2itnZ3k/Unesa7TAQDAOlvNFeBzk+xZ9Pxwa22y9/7Motf+\nYZI/6b0/vtKbbd26OZOTE89yzLWZmtp97PH09NRAP5vBs+Ma7LkGex59dlzDqbbn1QTw3iSLpx4/\nLn6T5M1J3riaD5yZ2b/K0dbP7OyBY4937Zod+OczONPTU3ZcgD3XYM+jz45rGOaelwrv1dwCsSPJ\n9UmycA/wA4sPttbOS/Kc3vt31jgjAABsuNVcAb4zyXWttXuTjCW5qbV2S5KdvfdPJXlRkkc2bkQA\nAFg/KwZw7/1Ikrcd9/JDi47fn/mfFAEAAKc8X4QBAEApAhgAgFIEMAAApQhgAABKEcAAAJQigAEA\nKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAAAEoRwAAAlCKAAQAoRQADAFCKAAYAoBQBDABA\nKQIYAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEMAEApAhgAgFIEMAAApQhgAABK\nEcAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAAAEoRwAAAlCKAAQAoRQADAFCK\nAAYAoBQBDABAKQIYAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUMrkSie01saT3J7kiiSHktzc\ne9+56Phrk7xv4enXk7y99z63AbMCAMCareYK8A1JNvXer0nyniS3Hj3QWptK8qEkv9J7357kkSQX\nbMCcAACwLlYTwNcmuStJeu/3Jblq0bGXJ3kgya2ttS8n+X7vfde6TwkAAOtkxVsgkpybZM+i54db\na5O992cyf7X3F5JcmWRfki+31r7Se//WUm+2devmTE5OrGXmZ21qavexx9PTUwP9bAbPjmuw5xrs\nefTZcQ2n2p5XE8B7kyyeenwhfpPkB0nu770/liSttS9lPoaXDOCZmf0nOerJm509cOzxrl2zA/98\nBmd6esqOC7DnGux59NlxDcPc81LhvZpbIHYkuT5JWmvbM3/Lw1FfS3J5a+2C1tpkku1JvrG2UQEA\nYOOs5grwnUmua63dm2QsyU2ttVuS7Oy9f6q19t4kdy+c+4ne+4MbNCsAAKzZigHcez+S5G3HvfzQ\nouN/nOSP13kuAADYEL4IAwCAUgQwAAClCGAAAEoRwAAAlCKAAQAoRQADAFCKAAYAoBQBDABAKQIY\nAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEMAEApAhgAgFIEMAAApQhgAABKEcAA\nAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAAAEoRwAAAlCKAAQAoRQADAFCKAAYA\noBQBDABAKQIYAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEMAEApAhgAgFIEMAAA\npQhgAABKEcAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCmTK53QWhtPcnuSK5IcSnJz733nouMfSfJz\nSWYXXnp9733PBswKAABrtmIAJ7khyabe+zWtte1Jbk3y+kXHfzbJL/XeH9+IAQEAYD2t5haIa5Pc\nlSS99/uSXHX0wMLV4Rcm+WhrbUdr7dc2ZEoAAFgnq7kCfG6Sxbc0HG6tTfben0lydpJ/m+S3k0wk\n+bPW2n/vvf/Ppd5s69bNmZycWMvMz9rU1O5jj6enpwb62QyeHddgzzXY8+iz4xpOtT2vJoD3Jlk8\n9fhC/CbJ/iS/23vfnySttS9k/l7hJQN4Zmb/SY568mZnDxx7vGvX7DJncrqbnp6y4wLsuQZ7Hn12\nXMMw97xUeK/mFogdSa5PkoV7gB9YdOxFSe5prU201s7I/O0SX1/bqAAAsHFWcwX4ziTXtdbuTTKW\n5KbW2i1JdvbeP9Va+8Mk9yV5Osl/6r3/r40bFwAA1mbFAO69H0nytuNefmjR8Q8m+eA6zwUAABvC\nF2EAAFCKAAYAoBQBDABAKQIYAIBSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEMAEAp\nAhgAgFIEMAAApQhgAABKEcAAAJRSIoBfcNF5efFPbMlLLt027FEAABiyEgH83K2b864bfzYXP3dq\n2KMAADBkJQIYAACOEsAAAJQigAEAKEUAAwBQigAGAKAUAQwAQCkCGACAUgQwAAClCGAAAEoRwAAA\nlCKAAQAoRQADAFCKAAYAoJSxubm5Yc8AAAAD4wowAAClCGAAAEoRwAAAlCKAAQAoRQADAFCKAAYA\noJTJYQ+w3lpr40luT3JFkkNJbu6971x0/C1J3prkmSS/2Xv/zFAGZU1Wsed/luTvLjz9XO/9/YOf\nkrVYaceLzvlskk/23v/94KdkrVbxa/m1Sd638PTrSd7ee/fzO08zq9jzO5O8KcmRJB/ovd85lEFZ\ns9bay5L8m977zx/3+uuS/KvM99fHeu//YQjjHTOKV4BvSLKp935NkvckufXogdbahUnekeTnkvxS\nkt9qrT1nKFOyVsvt+bIkb07y8iTXJHlNa+2nhzIla7Hkjhf5zSTbBjoV6225X8tTST6U5Fd679uT\nPJLkgmEMyZott+ctmf9v8zVJXpPktqFMyJq11t6V5PeSbDru9TOS/E7m9/uqJP9oocmGZhQD+Nok\ndyVJ7/2+JFctOvbSJDt674d673uS7EwijE5Py+35O0l+ufd+uPd+JMkZSQ4OfkTWaLkdp7X2xsxf\nLfr84EdjHS2355cneSDJra21Lyf5fu991+BHZB0st+cnkzya5OyFf44MfDrWy7eTvOEEr78kyc7e\n+0zv/akk9yR5xUAnO84oBvC5SfYsen64tTa5xLHZJOcNajDW1ZJ77r0/3Xt/vLU21lr7cJK/6L1/\nayhTshZL7ri1dnmSGzP/x2mc3pb7PfuCJL+Q5N1JXpvkn7bWXjTg+Vgfy+05mb9w8Y3M3+bykUEO\nxvrpvd+R5OkTHDrl+msUA3hvkqlFz8d7788scWwqye5BDca6Wm7Paa1tSvKHC+f8+oBnY30st+O/\nn+SiJF9I8g+S3NJa++XBjsc6WW7PP0hyf+/9sd77viRfSnLloAdkXSy359cm+fEkz0/yE0luaK29\ndMDzsbFOuf4axQDekeT6JGmtbc/8H58d9dUkr2itbWqtnZf5S/IPDn5E1sGSe26tjSX5ZJL/0Xt/\na+/98HBGZI2W3HHv/V2995ct/CWL/5jkt3vvdw1jSNZsud+zv5bk8tbaBQtXC7dn/iohp5/l9jyT\n5ECSQ733g5kPoy0Dn5CN9M0kL2ytbWutnZnklUm+MsyBRu6nQCS5M8l1rbV7k4wluam1dkvm7z35\nVGvtI0m+nPn4/5cLv9g4/Sy55yQTmb/J/jkLf4M8Sd7bex/qLzaetWV/LQ93NNbRSr9nvzfJ3Qvn\nfqL37qLF6WmlPb86yX2ttSOZvz/0T4c4K+uktXZjknN67x9d2Pfdme+vj/Xe/3qYs43NzflpMgAA\n1DGKt0AAAMCSBDAAAKUIYAAAShHAAACUIoABAChFAAMAUIoABgCgFAEMAEAp/x+K7vH0/AjSIQAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#导入ROC模块\n",
    "from sklearn.metrics import roc_curve #ROC曲线\n",
    "from sklearn.metrics import roc_auc_score #AUC分数\n",
    "\n",
    "fpr_rf,tpr_rf,thresholds_rf = roc_curve(y_test,model.predict_proba(x_test)[:,1])\n",
    "\n",
    "#找到最接近于0的阈值\n",
    "close_zero = np.argmin(np.abs(thresholds-0.5))\n",
    "plt.figure(figsize=(12,6))\n",
    "plt.plot(fpr_rf,tpr_rf,label=\"ROC\")\n",
    "plt.plot(fpr_rf[close_zero],tpr_rf[close_zero],\"o\",markersize=10,label=\"threshold 0.5rf\",fillstyle=\"none\",c=\"m\",mew=3)\n",
    "#打印AUC分数\n",
    "rf_auc = roc_auc_score(y_test,model.predict_proba(x_test)[:,1])\n",
    "print(\"AUC分数:\",rf_auc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
